Browse Source

feat: modify VAFE to keep only n expanded Features

The problem that I had overlooked until now (!!) due to only using small
exemplar videos for testing was that VideoActivityFeatureExtractor keeps
a fraction of small (0.5s) Features. This is not so problematic for a short
exemplar video, but ends up witha  lot of jumpy Features in an actual source
video.

Fix approach:

Like LoudnessFE, keep a specific number and expand the duration (which I think I
will do for LoudnessFE too), dropping any that would be consumed in that range
main
Rob Hallam 1 month ago
parent
commit
72fc6b563b
1 changed files with 36 additions and 2 deletions
  1. +36
    -2
      pipeline/feature_extractors.py

+ 36
- 2
pipeline/feature_extractors.py View File

@@ -296,12 +296,18 @@ class VideoActivityFeatureExtractor(FeatureExtractor):

#TODO: minimum duration -- consider whether to do here, or expand duration post-consolidation
"""
def __init__(self, input_files=None, config=None):
_CONFIG_DEFAULT_NUM_FEATURES = 5 # keep the top 5 activity moments
_CONFIG_DEFAULT_MIN_DURATION = 5.00 # seconds
def __init__(self, input_files=None, config=None,
num_features=_CONFIG_DEFAULT_NUM_FEATURES,
min_duration=_CONFIG_DEFAULT_MIN_DURATION):
if not input_files:
raise ValueError("No input files provided!")
self.input_files = input_files
self.config = config
self.features = []
self._num_features = num_features
self._min_duration = min_duration

def _scdet(self, video_file):
"""Run scdet filter on the video file"""
@@ -347,6 +353,29 @@ class VideoActivityFeatureExtractor(FeatureExtractor):
scores = sorted(scores, key=lambda x: x[1], reverse=True)
return scores[:int(len(scores) * (percent / 100))]

def _keep_num(self, features, num=_CONFIG_DEFAULT_NUM_FEATURES) -> list:
"""Keep the top n activity features (default: 5)

Approach:
- for range in 0-n
+ expand the nth top feature to min duration
(move start back by 0.5*min_duration, end forward by 0.5*min_duration)
+ drop any features that are now in that feature's range
- return the top n features

Each feature is a Feature object, with an Interval object
"""
for i in range(num):
# expand the feature to min_duration
features[i].interval.move_start(-0.5*self._min_duration, relative=True)
features[i].interval.move_end(0.5*self._min_duration, relative=True)
# drop any features that are now in that feature's range
features = [f for f in features if
f.interval.start < features[i].interval.start or
f.interval.end > features[i].interval.end]

return features[:num]

def setup(self):
pass

@@ -354,11 +383,16 @@ class VideoActivityFeatureExtractor(FeatureExtractor):
for file in self.input_files:
scores = self._scdet(file.path)
means = sorted(self._nonoverlap_mean(scores), key=lambda x: x[1], reverse=True)

features = []
for time, score in self._drop_lowest(means, 66):
self.features.append(Feature(interval=Interval(start=time, duration=0.500),
features.append(Feature(interval=Interval(start=time, duration=0.500),
source=file, feature_extractor="videoactivity",
score=score))

# prune features list to keep self.num_features
self.features = self._keep_num(features, self._num_features)

def teardown(self):
pass



Loading…
Cancel
Save