From 0d2096aed60766c5be84bd2e4db108e3d0c07fcb Mon Sep 17 00:00:00 2001 From: Rob Hallam <0504004h@student.gla.ac.uk> Date: Fri, 23 Aug 2024 22:40:20 +0100 Subject: [PATCH 1/8] =?UTF-8?q?refactor!:=20rename=20video=5Fproducers=20?= =?UTF-8?q?=E2=86=92=20producers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since it no longer simply produces videos this name is preferred Also update tests to use updated module name --- pipeline/{video_producers.py => producers.py} | 0 test/test_producers.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename pipeline/{video_producers.py => producers.py} (100%) diff --git a/pipeline/video_producers.py b/pipeline/producers.py similarity index 100% rename from pipeline/video_producers.py rename to pipeline/producers.py diff --git a/test/test_producers.py b/test/test_producers.py index 5f7d4ce..d0af2cf 100644 --- a/test/test_producers.py +++ b/test/test_producers.py @@ -1,6 +1,6 @@ """test_producers.py -- test the producers in the pipeline (eg ffmpeg, visualisation, json)""" import unittest -import pipeline.video_producers as producers +import pipeline.producers as producers import test.mocks as mocks From 2e45486372f5ae4b8d03c137305953779c863591 Mon Sep 17 00:00:00 2001 From: Rob Hallam <0504004h@student.gla.ac.uk> Date: Thu, 29 Aug 2024 09:45:14 +0100 Subject: [PATCH 2/8] test: add TestVideoActivityFEFunctional Tests VAFE using a manually-created 30 seconds video with activity between 15 and 20 seconds. Relatively quick! --- test/test_feature_extractors_functional.py | 44 ++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 test/test_feature_extractors_functional.py diff --git a/test/test_feature_extractors_functional.py b/test/test_feature_extractors_functional.py new file mode 100644 index 0000000..83fcc3c --- /dev/null +++ b/test/test_feature_extractors_functional.py @@ -0,0 +1,44 @@ +"""test_feature_extractors_functional.py -- functional tests for feature extractors + +This module contains functional tests for FEs using crafted and/or generated media files +to verify that the FEs are working as expected: + + - laughter detection -- uses videos with laughs at known times + - video activity -- uses videos with visual activity at known times + - audio loudness -- uses videos with audio at known times + +etc. + +These tests are marked slow to avoid running them during normal test runs. +""" + +import unittest +import pipeline.feature_extractors as extractors +import test.mocks as mocks +class TestVideoActivityFEFunctional(unittest.TestCase): + """TestVisualActivityFEFunctional -- functional tests for visual activity feature extractor + """ + + def test_visual_activity_functional(self): + """Test visual activity feature extractor + + use: + - sample_videos/sample-manual-visualactivity.mp4 :: activity at 15-20s -- pass if activity detected anywhere in this range + """ + SAMPLE_VIDEO = "/home/robert/code/softdev2023-24/summerproject/highlights/test/sample_videos/sample-manual-visualactivity.mp4" + + START_TIME = 15 + END_TIME = 20 + # create mock source with the video + source = mocks.MockSource(path=SAMPLE_VIDEO) + + # create the feature extractor + testfe = extractors.VideoActivityFeatureExtractor(input_files=[source]) + testfe.setup() + testfe.run() + testfe.teardown() + + # check if the feature was extracted: + self.assertTrue(testfe.features) + # check if the feature interval is within the expected range + self.assertTrue(testfe.features[0].interval.start >= START_TIME) From 428032d841aab0fd1eb69c2b69b7ede1e58844f1 Mon Sep 17 00:00:00 2001 From: Rob Hallam <0504004h@student.gla.ac.uk> Date: Thu, 29 Aug 2024 09:47:14 +0100 Subject: [PATCH 3/8] test: add FEFunctionalTest base class for FE tests Only has a single property at present: SAMPLE_DIR for the path to where sample videos are stored TestVideoActivityFEFunctional now inherits from this instead of unittest.TestCase --- test/test_feature_extractors_functional.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/test_feature_extractors_functional.py b/test/test_feature_extractors_functional.py index 83fcc3c..b608bf5 100644 --- a/test/test_feature_extractors_functional.py +++ b/test/test_feature_extractors_functional.py @@ -15,7 +15,13 @@ These tests are marked slow to avoid running them during normal test runs. import unittest import pipeline.feature_extractors as extractors import test.mocks as mocks -class TestVideoActivityFEFunctional(unittest.TestCase): + +class FEFunctionalTest(unittest.TestCase): + """FEFunctionalTest -- base class for functional tests for feature extractors + """ + SAMPLE_DIR = "/home/robert/code/softdev2023-24/summerproject/highlights/test/sample_videos" + +class TestVideoActivityFEFunctional(FEFunctionalTest): """TestVisualActivityFEFunctional -- functional tests for visual activity feature extractor """ @@ -25,7 +31,7 @@ class TestVideoActivityFEFunctional(unittest.TestCase): use: - sample_videos/sample-manual-visualactivity.mp4 :: activity at 15-20s -- pass if activity detected anywhere in this range """ - SAMPLE_VIDEO = "/home/robert/code/softdev2023-24/summerproject/highlights/test/sample_videos/sample-manual-visualactivity.mp4" + SAMPLE_VIDEO = f"{self.SAMPLE_DIR}/sample-manual-visualactivity.mp4" START_TIME = 15 END_TIME = 20 From 192c952c2da014680a790448f3bfe2a296000039 Mon Sep 17 00:00:00 2001 From: Rob Hallam <0504004h@student.gla.ac.uk> Date: Thu, 29 Aug 2024 09:49:11 +0100 Subject: [PATCH 4/8] test: add TestLoudAudioFEFunctional Functional tests for LoudAudioFeatureExtractor Currently uses one manually-generated video with blank audio except between 15-20s where 1-2 sine tones are present --- test/test_feature_extractors_functional.py | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/test/test_feature_extractors_functional.py b/test/test_feature_extractors_functional.py index b608bf5..706b509 100644 --- a/test/test_feature_extractors_functional.py +++ b/test/test_feature_extractors_functional.py @@ -48,3 +48,45 @@ class TestVideoActivityFEFunctional(FEFunctionalTest): self.assertTrue(testfe.features) # check if the feature interval is within the expected range self.assertTrue(testfe.features[0].interval.start >= START_TIME) + + +class TestLoudAudioFEFunctional(FEFunctionalTest): + """TestAudioLoudnessFEFunctional -- functional tests for audio loudness feature extractor + """ + + def test_audio_loudness_functional_one_feature(self): + """Test audio loudness feature extractor + + use: + - sample_videos/sample-manual-audio.mp4 :: audio at 15-20s -- pass if audio detected anywhere in this range + -- peak at 16s - 18s, verify this is highest scoring + """ + SAMPLE_VIDEO = f"{self.SAMPLE_DIR}/sample-manual-audio.mp4" + + START_TIME = 15 + END_TIME = 20 + PEAK_START = 16 + PEAK_END = 18 + # create mock source with the video + source = mocks.MockSource(path=SAMPLE_VIDEO) + + # create the feature extractor + testfe = extractors.LoudAudioFeatureExtractor(input_files=[source]) + testfe.setup() + testfe.run() + testfe.teardown() + + # check if the feature was extracted: + self.assertTrue(testfe.features) + # check if the feature interval is within the expected range + self.assertTrue(testfe.features[0].interval.start >= START_TIME) + + # get sorted list of features based on feature.score + sorted_features = sorted(testfe.features, key=lambda x: x.score, reverse=True) + # check if the highest scoring feature is within the peak range + self.assertTrue(sorted_features[0].interval.start >= PEAK_START) + + + +if __name__ == "__main__": + unittest.main() From 8a5916192a5376b8270c36094ed7ea21efcf805a Mon Sep 17 00:00:00 2001 From: Rob Hallam <0504004h@student.gla.ac.uk> Date: Thu, 29 Aug 2024 09:50:50 +0100 Subject: [PATCH 5/8] test: add negative test to TestLoudAudioFEFunctional Video file with silent audio (generated with sox) -- should produce no features since "-inf" are filtered by the FE --- test/test_feature_extractors_functional.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/test_feature_extractors_functional.py b/test/test_feature_extractors_functional.py index 706b509..bd565b8 100644 --- a/test/test_feature_extractors_functional.py +++ b/test/test_feature_extractors_functional.py @@ -86,6 +86,28 @@ class TestLoudAudioFEFunctional(FEFunctionalTest): # check if the highest scoring feature is within the peak range self.assertTrue(sorted_features[0].interval.start >= PEAK_START) + def test_audio_loudness_functional_no_features(self): + """Test audio loudness feature extractor using a silent video. This should produce no features + since "-inf" results from pyloudnorm are filtered out by the FE. + + Use: + - sample_videos/sample-manual-audio-blank-video-colours.mp4 + :: silent video (30s) + -- pass if no features extracted + """ + SAMPLE_VIDEO = f"{self.SAMPLE_DIR}/sample-manual-audio-blank-video-colours.mp4" + + # create mock source with the video + source = mocks.MockSource(path=SAMPLE_VIDEO) + + # create the feature extractor + testfe = extractors.LoudAudioFeatureExtractor(input_files=[source]) + testfe.setup() + testfe.run() + testfe.teardown() + + # check if the feature was extracted: + self.assertFalse(testfe.features) if __name__ == "__main__": From df3c5599391422b72e53ebcd0170e1bcc7bdd623 Mon Sep 17 00:00:00 2001 From: Rob Hallam <0504004h@student.gla.ac.uk> Date: Thu, 29 Aug 2024 10:30:35 +0100 Subject: [PATCH 6/8] refactor: expose prepend and append tmies in LaughFE To help functional testing, LaughFE's internal adjustment times are exposed. Recap: when a laugh is detected by LaughFE, the time of the laugh itself it not used directly; instead, the resulting Feature has some time prepended to try to capture the thing that caused the laugh. When functional testing the FEs we set up specially-crafted videos with features at known points, so to make sure the LaughterFE is tested correctly we adjust the tests by the amount of time the FE adjusts by so that it properly tests the intended behaviour. @see: - feature_extractors.py::LaughterFeatureExtractor --- pipeline/feature_extractors.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pipeline/feature_extractors.py b/pipeline/feature_extractors.py index cf9dbdb..38063de 100644 --- a/pipeline/feature_extractors.py +++ b/pipeline/feature_extractors.py @@ -47,6 +47,9 @@ class LaughterFeatureExtractor(FeatureExtractor): See: https://github.com/jrgillick/laughter-detection for the laughter-detection library """ + _PREPEND_TIME = 7.0 # seconds before the laugh + _APPEND_TIME = 3.0 # seconds after the laugh + def __init__(self, input_files=None, config=None): """It is expected that input_files is a SourceMedia object""" @@ -94,13 +97,10 @@ class LaughterFeatureExtractor(FeatureExtractor): TODO: config for length adjustments per design doc TODO: play with numbers more to see what works best """ - PREPEND = 7.0 - APPEND = 3.0 - for feature in self.features: # do the pre & post adjustment - feature.interval.move_start(-PREPEND, relative=True) - feature.interval.move_end(APPEND, relative=True) + feature.interval.move_start(-self._PREPEND_TIME, relative=True) + feature.interval.move_end(self._APPEND_TIME, relative=True) def setup(self): """Setup the laughter feature extractor -- validate input files & config From 9af169f6370b23e098229bbf737b0b266a80287e Mon Sep 17 00:00:00 2001 From: Rob Hallam <0504004h@student.gla.ac.uk> Date: Thu, 29 Aug 2024 10:50:19 +0100 Subject: [PATCH 7/8] test: add TestLaughterFEFunctional Uses a manually-crafted video with laughters between 15-20s. Test takes LaughFE's internal Feature time adjustment into account (see related commit). Note: very slow test @see: df3c559 --- test/test_feature_extractors_functional.py | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/test_feature_extractors_functional.py b/test/test_feature_extractors_functional.py index bd565b8..defcff2 100644 --- a/test/test_feature_extractors_functional.py +++ b/test/test_feature_extractors_functional.py @@ -21,6 +21,41 @@ class FEFunctionalTest(unittest.TestCase): """ SAMPLE_DIR = "/home/robert/code/softdev2023-24/summerproject/highlights/test/sample_videos" + +class TestLaughterFEFunctional(FEFunctionalTest): + """TestLaughterFEFunctional -- functional tests for laughter detection feature extractor""" + + def test_laughter_detection(self): + """Test laughter detection feature extractor + + Uses: + - sample_videos/sample-manual-audio-laughs-video-colours.mp4 + :: laughters at 15-20s + -- pass iff laughter features extracted in this range, *but* + NOTE: LaughFE subtracts from start time to capture what preceded the laughter + so we need to subtract this time (and adds a little after too) + FE 'exposes' these as _PREPEND_TIME and _APPEND_TIME + """ + SAMPLE_VIDEO = f"{self.SAMPLE_DIR}/sample-manual-audio-laughs-video-colours.mp4" + + START_TIME = 15 + END_TIME = 20 + # create mock source with the video + source = mocks.MockSource(path=SAMPLE_VIDEO) + + # create the feature extractor + testfe = extractors.LaughterFeatureExtractor(input_files=[source]) + testfe.setup() + testfe.run() + testfe.teardown() + + # check if the feature was extracted: + self.assertTrue(testfe.features) + # check if the feature interval is within the expected range + self.assertTrue(testfe.features[0].interval.start >= (START_TIME - testfe._PREPEND_TIME)) + self.assertTrue(testfe.features[0].interval.end <= (END_TIME + testfe._APPEND_TIME)) + + class TestVideoActivityFEFunctional(FEFunctionalTest): """TestVisualActivityFEFunctional -- functional tests for visual activity feature extractor """ From 1c9a6a6329e53d795b3a36588f48961b2ba2d7e8 Mon Sep 17 00:00:00 2001 From: Rob Hallam <0504004h@student.gla.ac.uk> Date: Thu, 29 Aug 2024 10:58:28 +0100 Subject: [PATCH 8/8] test: [fefunctional] add pytest and mark LaughFE slow&verslow Also add note that it takes 8-10s to run a single LaughFE test --- test/test_feature_extractors_functional.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_feature_extractors_functional.py b/test/test_feature_extractors_functional.py index defcff2..e3736f0 100644 --- a/test/test_feature_extractors_functional.py +++ b/test/test_feature_extractors_functional.py @@ -12,6 +12,7 @@ etc. These tests are marked slow to avoid running them during normal test runs. """ +import pytest import unittest import pipeline.feature_extractors as extractors import test.mocks as mocks @@ -22,6 +23,8 @@ class FEFunctionalTest(unittest.TestCase): SAMPLE_DIR = "/home/robert/code/softdev2023-24/summerproject/highlights/test/sample_videos" +@pytest.mark.slow +@pytest.mark.veryslow class TestLaughterFEFunctional(FEFunctionalTest): """TestLaughterFEFunctional -- functional tests for laughter detection feature extractor""" @@ -35,6 +38,9 @@ class TestLaughterFEFunctional(FEFunctionalTest): NOTE: LaughFE subtracts from start time to capture what preceded the laughter so we need to subtract this time (and adds a little after too) FE 'exposes' these as _PREPEND_TIME and _APPEND_TIME + + Note: takes 8-10s to run for this 30s video using GTX 970. As such this test can be skipped with either: + "-m 'not veryslow'" or "-m 'not slow'" """ SAMPLE_VIDEO = f"{self.SAMPLE_DIR}/sample-manual-audio-laughs-video-colours.mp4"