Browse Source

feat: modify VAFE to keep only n expanded Features

The problem that I had overlooked until now (!!) due to only using small
exemplar videos for testing was that VideoActivityFeatureExtractor keeps
a fraction of small (0.5s) Features. This is not so problematic for a short
exemplar video, but ends up witha  lot of jumpy Features in an actual source
video.

Fix approach:

Like LoudnessFE, keep a specific number and expand the duration (which I think I
will do for LoudnessFE too), dropping any that would be consumed in that range
main
Rob Hallam 1 month ago
parent
commit
72fc6b563b
1 changed files with 36 additions and 2 deletions
  1. +36
    -2
      pipeline/feature_extractors.py

+ 36
- 2
pipeline/feature_extractors.py View File

@@ -296,12 +296,18 @@ class VideoActivityFeatureExtractor(FeatureExtractor):


#TODO: minimum duration -- consider whether to do here, or expand duration post-consolidation #TODO: minimum duration -- consider whether to do here, or expand duration post-consolidation
""" """
def __init__(self, input_files=None, config=None):
_CONFIG_DEFAULT_NUM_FEATURES = 5 # keep the top 5 activity moments
_CONFIG_DEFAULT_MIN_DURATION = 5.00 # seconds
def __init__(self, input_files=None, config=None,
num_features=_CONFIG_DEFAULT_NUM_FEATURES,
min_duration=_CONFIG_DEFAULT_MIN_DURATION):
if not input_files: if not input_files:
raise ValueError("No input files provided!") raise ValueError("No input files provided!")
self.input_files = input_files self.input_files = input_files
self.config = config self.config = config
self.features = [] self.features = []
self._num_features = num_features
self._min_duration = min_duration


def _scdet(self, video_file): def _scdet(self, video_file):
"""Run scdet filter on the video file""" """Run scdet filter on the video file"""
@@ -347,6 +353,29 @@ class VideoActivityFeatureExtractor(FeatureExtractor):
scores = sorted(scores, key=lambda x: x[1], reverse=True) scores = sorted(scores, key=lambda x: x[1], reverse=True)
return scores[:int(len(scores) * (percent / 100))] return scores[:int(len(scores) * (percent / 100))]


def _keep_num(self, features, num=_CONFIG_DEFAULT_NUM_FEATURES) -> list:
"""Keep the top n activity features (default: 5)

Approach:
- for range in 0-n
+ expand the nth top feature to min duration
(move start back by 0.5*min_duration, end forward by 0.5*min_duration)
+ drop any features that are now in that feature's range
- return the top n features

Each feature is a Feature object, with an Interval object
"""
for i in range(num):
# expand the feature to min_duration
features[i].interval.move_start(-0.5*self._min_duration, relative=True)
features[i].interval.move_end(0.5*self._min_duration, relative=True)
# drop any features that are now in that feature's range
features = [f for f in features if
f.interval.start < features[i].interval.start or
f.interval.end > features[i].interval.end]

return features[:num]

def setup(self): def setup(self):
pass pass


@@ -354,11 +383,16 @@ class VideoActivityFeatureExtractor(FeatureExtractor):
for file in self.input_files: for file in self.input_files:
scores = self._scdet(file.path) scores = self._scdet(file.path)
means = sorted(self._nonoverlap_mean(scores), key=lambda x: x[1], reverse=True) means = sorted(self._nonoverlap_mean(scores), key=lambda x: x[1], reverse=True)

features = []
for time, score in self._drop_lowest(means, 66): for time, score in self._drop_lowest(means, 66):
self.features.append(Feature(interval=Interval(start=time, duration=0.500),
features.append(Feature(interval=Interval(start=time, duration=0.500),
source=file, feature_extractor="videoactivity", source=file, feature_extractor="videoactivity",
score=score)) score=score))


# prune features list to keep self.num_features
self.features = self._keep_num(features, self._num_features)

def teardown(self): def teardown(self):
pass pass




Loading…
Cancel
Save