<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD JATS (Z39.96) Journal Publishing DTD v1.3 20210610//EN" "JATS-journalpublishing1-3.dtd">
<article article-type="research-article" dtd-version="1.3" xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xml:lang="ru"><front><journal-meta><journal-id journal-id-type="publisher-id">novtexmech</journal-id><journal-title-group><journal-title xml:lang="ru">Мехатроника, автоматизация, управление</journal-title><trans-title-group xml:lang="en"><trans-title>Mekhatronika, Avtomatizatsiya, Upravlenie</trans-title></trans-title-group></journal-title-group><issn pub-type="ppub">1684-6427</issn><issn pub-type="epub">2619-1253</issn><publisher><publisher-name>Commercial Publisher «New Technologies»</publisher-name></publisher></journal-meta><article-meta><article-id custom-type="elpub" pub-id-type="custom">novtexmech-28</article-id><article-categories><subj-group subj-group-type="heading"><subject>Research Article</subject></subj-group><subj-group subj-group-type="section-heading" xml:lang="ru"><subject>ИНТЕРФЕЙСЫ ЭРГАТИЧЕСКИХ СИСТЕМ УПРАВЛЕНИЯ</subject></subj-group><subj-group subj-group-type="section-heading" xml:lang="en"><subject>INTERFACES OF ERGATE CONTROL SYSTEMS</subject></subj-group></article-categories><title-group><article-title>Аудиовизуальный детектор голосовой активности на базе глубокой сверточной сети и обобщенной взаимной корреляции</article-title><trans-title-group xml:lang="en"><trans-title>Audiovisual Voice Activity Detector Based on Deep Convolutional Neural Network and Generalized Cross-Correlation</trans-title></trans-title-group></title-group><contrib-group><contrib contrib-type="author" corresp="yes"><name-alternatives><name name-style="eastern" xml:lang="ru"><surname>Суворов</surname><given-names>Д. А.</given-names></name><name name-style="western" xml:lang="en"><surname>Suvorov</surname><given-names>D. A.</given-names></name></name-alternatives><email xlink:type="simple">dmitry.suvorov@skolkovotech.ru</email><xref ref-type="aff" rid="aff-1"/></contrib><contrib contrib-type="author" corresp="yes"><name-alternatives><name name-style="eastern" xml:lang="ru"><surname>Жуков</surname><given-names>Р. А.</given-names></name><name name-style="western" xml:lang="en"><surname>Zhukov</surname><given-names>R. A.</given-names></name></name-alternatives><email xlink:type="simple">roman.zhukov@skolkovotech.ru</email><xref ref-type="aff" rid="aff-1"/></contrib><contrib contrib-type="author" corresp="yes"><name-alternatives><name name-style="eastern" xml:lang="ru"><surname>Тетерюков</surname><given-names>Д. О.</given-names></name><name name-style="western" xml:lang="en"><surname>Tsetserukov</surname><given-names>D. O.</given-names></name></name-alternatives><email xlink:type="simple">d.tsetserukou@skoltech.ru</email><xref ref-type="aff" rid="aff-1"/></contrib><contrib contrib-type="author" corresp="yes"><name-alternatives><name name-style="eastern" xml:lang="ru"><surname>Зенкевич</surname><given-names>С. Л.</given-names></name><name name-style="western" xml:lang="en"><surname>Zenkevich</surname><given-names>S. L.</given-names></name></name-alternatives><email xlink:type="simple">zenkev@bmstu.ru</email><xref ref-type="aff" rid="aff-2"/></contrib></contrib-group><aff-alternatives id="aff-1"><aff xml:lang="ru"><institution>Сколковский институт науки и технологий</institution><country>Россия</country></aff><aff xml:lang="en"><institution>Skolkovo Institute of Science and Technology</institution><country>Russian Federation</country></aff></aff-alternatives><aff-alternatives id="aff-2"><aff xml:lang="ru"><institution>Московский государственный технический университет имени Н. Э. Баумана</institution><country>Россия</country></aff><aff xml:lang="en"><institution>Bauman Moscow State Technical University</institution><country>Russian Federation</country></aff></aff-alternatives><pub-date pub-type="collection"><year>2018</year></pub-date><pub-date pub-type="epub"><day>23</day><month>08</month><year>2018</year></pub-date><volume>19</volume><issue>1</issue><fpage>53</fpage><lpage>57</lpage><permissions><copyright-statement>Copyright &amp;#x00A9; Commercial Publisher «New Technologies», 2018</copyright-statement><copyright-year>2018</copyright-year><copyright-holder xml:lang="ru">Commercial Publisher «New Technologies»</copyright-holder><copyright-holder xml:lang="en">Commercial Publisher «New Technologies»</copyright-holder><license xlink:href="https://mech.novtex.ru/jour/about/submissions#copyrightNotice" xlink:type="simple"><license-p>https://mech.novtex.ru/jour/about/submissions#copyrightNotice</license-p></license></permissions><self-uri xlink:href="https://mech.novtex.ru/jour/article/view/28">https://mech.novtex.ru/jour/article/view/28</self-uri><abstract><p>Разработан алгоритм детектора голосовой активности, использующий данные с видеокамеры и массива микрофонов и благодаря этому обладающий высокой устойчивостью к внешним шумам. Обработка видеокадров заключается в поиске губ человека с помощью глубокой сверточной нейронной сети, обработка звука - в локализации источников звука с помощью обобщенной функции взаимной корреляции с весовой функцией преобразования фазы (GCC-PHAT). Решение об активации детектора голосовой активности принимается только в случае нахождения соответствия между направлением на губы и на активные источники звука. Разработанный детектор показал высокую устойчивость к шумам - шумы, производимые источниками звука вне видимости видеокамеры или целевого сектора для массива микрофонов, игнорировались в 100 % случаев во время эксперимента.</p></abstract><trans-abstract xml:lang="en"><p>This paper presents a voice activity detector (VAD) which uses the data from the compact linear microphone array and a video camera, so developed VAD is robust to external noise conditions. It is able to ignore non-speech sound sources and speaking persons located out of the area of the interest. A deep convolutional neural network processes images from the video camera for searching face and lips of the speaking person. It was trained using the Max-Margin Object Detection loss. Pixel coordinates of found lips are converting to directions to lips in camera coordinate system using optical camera model. The sound from the microphone array is processing using the weighted GCC-PHAT algorithm and Kalman filtering. VAD searches for speaking lips on the video. It becomes activated only if the video camera finds lips and the microphone array confirms that there is a sound source in this direction. A prototype of the system based the linear microphone array with 30 mm spacing between microphones and the video camera was developed, manufactured using a 3D printer and tested in the laboratory conditions. The accuracy of the system was compared with the open source VAD from the WebRTC project (developed by Google) which uses only audio features extracted from the same microphone array. Developed VAD showed a high sustainability to external noise. It ignored the noise from not-target directions during 100 % of the testing time. And the VAD from the WebRTC had 88 % of false positive activations.</p></trans-abstract><kwd-group xml:lang="ru"><kwd>детектор голосовой активности</kwd><kwd>массив микрофонов</kwd><kwd>сверточные сети</kwd><kwd>локализация источников звука</kwd><kwd>обработка звука</kwd><kwd>voice activity detector</kwd><kwd>microphone array</kwd><kwd>convolutional networks</kwd></kwd-group></article-meta></front><back><ref-list><title>References</title><ref id="cit1"><label>1</label><citation-alternatives><mixed-citation xml:lang="ru">RamHrez J., Gorriz J. M., Segura J. C. Voice activity detection. Fundamentals and speech recognition system robustness // Robust Speech Recognition and Understanding. Vienna: I-TECH Education and Publishing. 2007. P. 1-22.</mixed-citation><mixed-citation xml:lang="en">RamHrez J., Gorriz J. M., Segura J. C. Voice activity detection. Fundamentals and speech recognition system robustness // Robust Speech Recognition and Understanding. Vienna: I-TECH Education and Publishing. 2007. P. 1-22.</mixed-citation></citation-alternatives></ref><ref id="cit2"><label>2</label><citation-alternatives><mixed-citation xml:lang="ru">Woo K., Yang T., Park K., Lee C. Robust voice activity detection algorithm for estimating noise spectrum // Electronics Letters. 2000. Vol. 36, N. 2. P. 180-181.</mixed-citation><mixed-citation xml:lang="en">Woo K., Yang T., Park K., Lee C. Robust voice activity detection algorithm for estimating noise spectrum // Electronics Letters. 2000. Vol. 36, N. 2. P. 180-181.</mixed-citation></citation-alternatives></ref><ref id="cit3"><label>3</label><citation-alternatives><mixed-citation xml:lang="ru">Mousazadeh S., Cohen I. Voice activity detection in presence of transient noise using spectral clustering // IEEE Trans. Audio, Speech, Language Process. 2013. Vol. 21, N. 6. P. 1261-1271.</mixed-citation><mixed-citation xml:lang="en">Mousazadeh S., Cohen I. Voice activity detection in presence of transient noise using spectral clustering // IEEE Trans. Audio, Speech, Language Process. 2013. Vol. 21, N. 6. P. 1261-1271.</mixed-citation></citation-alternatives></ref><ref id="cit4"><label>4</label><citation-alternatives><mixed-citation xml:lang="ru">Obuchi Y. Framewise speech-nonspeech classification by neural networks for voice activity detection with statistical noise suppression // IEEE International Conference on Acoustics, Speech and Signal Processing. Shanghai, 2016, March. P. 5715-5719.</mixed-citation><mixed-citation xml:lang="en">Obuchi Y. Framewise speech-nonspeech classification by neural networks for voice activity detection with statistical noise suppression // IEEE International Conference on Acoustics, Speech and Signal Processing. Shanghai, 2016, March. P. 5715-5719.</mixed-citation></citation-alternatives></ref><ref id="cit5"><label>5</label><citation-alternatives><mixed-citation xml:lang="ru">Montazzolli S., Jung C. R., Gelb D. Audiovisual voice activity detection using off-the-shelf cameras // IEEE International Conference on Image Processing. Quebec, 2015, September. P. 3886-3890.</mixed-citation><mixed-citation xml:lang="en">Montazzolli S., Jung C. R., Gelb D. Audiovisual voice activity detection using off-the-shelf cameras // IEEE International Conference on Image Processing. Quebec, 2015, September. P. 3886-3890.</mixed-citation></citation-alternatives></ref><ref id="cit6"><label>6</label><citation-alternatives><mixed-citation xml:lang="ru">Ying D., Yan Y., Dang J., Soong F. K. Voice activity detection based on an unsupervised learning framework // IEEE Trans. Audio, Speech, Language Process. 2011. Vol. 19, N. 8. P. 2624-2633.</mixed-citation><mixed-citation xml:lang="en">Ying D., Yan Y., Dang J., Soong F. K. Voice activity detection based on an unsupervised learning framework // IEEE Trans. Audio, Speech, Language Process. 2011. Vol. 19, N. 8. P. 2624-2633.</mixed-citation></citation-alternatives></ref><ref id="cit7"><label>7</label><citation-alternatives><mixed-citation xml:lang="ru">Popovic B., Pakoci E., Pekar D. Advanced Voice Activity Detection on Mobile Phones by Using Microphone Array and Phoneme-Specific Gaussian Mixture Models // SISY. Subotica, 2016, August. P. 45-50.</mixed-citation><mixed-citation xml:lang="en">Popovic B., Pakoci E., Pekar D. Advanced Voice Activity Detection on Mobile Phones by Using Microphone Array and Phoneme-Specific Gaussian Mixture Models // SISY. Subotica, 2016, August. P. 45-50.</mixed-citation></citation-alternatives></ref><ref id="cit8"><label>8</label><citation-alternatives><mixed-citation xml:lang="ru">Grondin F., Michaud F. Noise Mask for TDOA Sound Source Localization of Speech on Mobile Robots in Noisy Environments // IEEE International Conference Robotics and Automation. Stockholm, 2016, May.</mixed-citation><mixed-citation xml:lang="en">Grondin F., Michaud F. Noise Mask for TDOA Sound Source Localization of Speech on Mobile Robots in Noisy Environments // IEEE International Conference Robotics and Automation. Stockholm, 2016, May.</mixed-citation></citation-alternatives></ref><ref id="cit9"><label>9</label><citation-alternatives><mixed-citation xml:lang="ru">Tashev I., Mirsamadi S. DNN-based Causal Voice Activity Detector // Information Theory and Applications Workshop. San Diego, 2016, February.</mixed-citation><mixed-citation xml:lang="en">Tashev I., Mirsamadi S. DNN-based Causal Voice Activity Detector // Information Theory and Applications Workshop. San Diego, 2016, February.</mixed-citation></citation-alternatives></ref><ref id="cit10"><label>10</label><citation-alternatives><mixed-citation xml:lang="ru">Julier S., Uhlmann J. A new extension of the Kalman filter to nonlinear systems // 11th International Symposium on Aerospace/ Defense Sensing, Simulation and Controls. Vol. Multi- Sensor Fusion, Tracking and Resource Management II. Orlando, 1997.</mixed-citation><mixed-citation xml:lang="en">Julier S., Uhlmann J. A new extension of the Kalman filter to nonlinear systems // 11th International Symposium on Aerospace/ Defense Sensing, Simulation and Controls. Vol. Multi- Sensor Fusion, Tracking and Resource Management II. Orlando, 1997.</mixed-citation></citation-alternatives></ref><ref id="cit11"><label>11</label><citation-alternatives><mixed-citation xml:lang="ru">King D. E. Max-Margin Object Detection // Cornell University Library. 31.12.15. URL: https://arxiv.org/pdf/1502.00046.pdf (дата обращения: 18.08.2017).</mixed-citation><mixed-citation xml:lang="en">King D. E. Max-Margin Object Detection // Cornell University Library. 31.12.15. URL: https://arxiv.org/pdf/1502.00046.pdf (дата обращения: 18.08.2017).</mixed-citation></citation-alternatives></ref><ref id="cit12"><label>12</label><citation-alternatives><mixed-citation xml:lang="ru">Kazemi V., Sullivan J. One Millisecond Face Alignment with an Ensemble of Regression Trees // IEEE Conference on Computer Vision and Pattern Recognition. Columbus, 2014, June.</mixed-citation><mixed-citation xml:lang="en">Kazemi V., Sullivan J. One Millisecond Face Alignment with an Ensemble of Regression Trees // IEEE Conference on Computer Vision and Pattern Recognition. Columbus, 2014, June.</mixed-citation></citation-alternatives></ref><ref id="cit13"><label>13</label><citation-alternatives><mixed-citation xml:lang="ru">Bradski G., Kaehler A. Learning OpenCV. Computer Vision with the OpenCV Library. Sebastopol: O'Reilly Media, 2008. P. 580.</mixed-citation><mixed-citation xml:lang="en">Bradski G., Kaehler A. Learning OpenCV. Computer Vision with the OpenCV Library. Sebastopol: O'Reilly Media, 2008. P. 580.</mixed-citation></citation-alternatives></ref><ref id="cit14"><label>14</label><citation-alternatives><mixed-citation xml:lang="ru">Tashev I. Sound Capture and Processing. Practical Approaches. The City of New York: John Wiley &amp; Sons, 2009. P. 365.</mixed-citation><mixed-citation xml:lang="en">Tashev I. Sound Capture and Processing. Practical Approaches. The City of New York: John Wiley &amp; Sons, 2009. P. 365.</mixed-citation></citation-alternatives></ref><ref id="cit15"><label>15</label><citation-alternatives><mixed-citation xml:lang="ru">Суворов Д. А., Жуков Р. А. Устройство синхронного сбора данных с массива MEMS микрофонов с PDM интерфейсом. Патент России № 172596. 2017. Бюл. № 20.</mixed-citation><mixed-citation xml:lang="en">Суворов Д. А., Жуков Р. А. Устройство синхронного сбора данных с массива MEMS микрофонов с PDM интерфейсом. Патент России № 172596. 2017. Бюл. № 20.</mixed-citation></citation-alternatives></ref></ref-list><fn-group><fn fn-type="conflict"><p>The authors declare that there are no conflicts of interest present.</p></fn></fn-group></back></article>
