From a672ab5e737202bede956a88357a96cf2728df15 Mon Sep 17 00:00:00 2001 From: Joerg Mueller Date: Tue, 9 Aug 2011 14:10:32 +0000 Subject: 3D Audio GSoC: Improved waveform drawing in the sequencer. * Drawing the waveform of a sequencer strip is now independent from whether the sound is cached or not. * Improved drawing of the waveform in the sequencer (especially speed!). * Making it possible to vertically zoom more in the sequencer to better see the waveform for lipsync. * Fixed a bug which crashed blender on loading a sound file via ffmpeg. --- intern/audaspace/ffmpeg/AUD_FFMPEGReader.cpp | 1 + intern/audaspace/intern/AUD_C-API.cpp | 38 +++++++------- intern/audaspace/intern/AUD_C-API.h | 2 +- release/scripts/startup/bl_ui/space_sequencer.py | 1 + source/blender/blenkernel/BKE_sound.h | 12 ++++- source/blender/blenkernel/intern/sound.c | 35 +++++++++++-- source/blender/blenloader/intern/readfile.c | 31 +++++++++++ .../editors/space_sequencer/sequencer_draw.c | 61 +++++++++++++++++----- .../editors/space_sequencer/space_sequencer.c | 2 +- source/blender/makesdna/DNA_sequence_types.h | 1 + source/blender/makesdna/DNA_sound_types.h | 5 ++ source/blender/makesrna/intern/rna_sequencer.c | 5 ++ 12 files changed, 154 insertions(+), 40 deletions(-) diff --git a/intern/audaspace/ffmpeg/AUD_FFMPEGReader.cpp b/intern/audaspace/ffmpeg/AUD_FFMPEGReader.cpp index b980e1d98e0..1683a9a61c0 100644 --- a/intern/audaspace/ffmpeg/AUD_FFMPEGReader.cpp +++ b/intern/audaspace/ffmpeg/AUD_FFMPEGReader.cpp @@ -177,6 +177,7 @@ static const char* fileopen_error = "AUD_FFMPEGReader: File couldn't be " AUD_FFMPEGReader::AUD_FFMPEGReader(std::string filename) : m_pkgbuf(AVCODEC_MAX_AUDIO_FRAME_SIZE<<1), + m_formatCtx(NULL), m_aviocontext(NULL), m_membuf(NULL) { diff --git a/intern/audaspace/intern/AUD_C-API.cpp b/intern/audaspace/intern/AUD_C-API.cpp index e5c966fdcae..85a053238d0 100644 --- a/intern/audaspace/intern/AUD_C-API.cpp +++ b/intern/audaspace/intern/AUD_C-API.cpp @@ -816,7 +816,7 @@ float* AUD_readSoundBuffer(const char* filename, float low, float high, if(high < rate) sound = new AUD_LowpassFactory(sound, high); if(low > 0) - sound = new AUD_HighpassFactory(sound, low);; + sound = new AUD_HighpassFactory(sound, low); sound = new AUD_EnvelopeFactory(sound, attack, release, threshold, 0.1f); sound = new AUD_LinearResampleFactory(sound, specs); @@ -1055,7 +1055,7 @@ int AUD_doesPlayback() return -1; } -int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length) +int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length, int samples_per_second) { AUD_DeviceSpecs specs; sample_t* buf; @@ -1067,38 +1067,40 @@ int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length) AUD_Reference reader = AUD_ChannelMapperFactory(*sound, specs).createReader(); - int len = reader->getLength(); - float samplejump = (float)len / (float)length; - float min, max; + specs.specs = reader->getSpecs(); + int len; + float samplejump = specs.rate / samples_per_second; + float min, max, power; bool eos; for(int i = 0; i < length; i++) { len = floor(samplejump * (i+1)) - floor(samplejump * i); - if(aBuffer.getSize() < len * AUD_SAMPLE_SIZE(reader->getSpecs())) - { - aBuffer.resize(len * AUD_SAMPLE_SIZE(reader->getSpecs())); - buf = aBuffer.getBuffer(); - } + aBuffer.assureSize(len * AUD_SAMPLE_SIZE(specs)); + buf = aBuffer.getBuffer(); reader->read(len, eos, buf); - if(eos) - { - length = i; - break; - } - max = min = *buf; + power = *buf * *buf; for(int j = 1; j < len; j++) { if(buf[j] < min) min = buf[j]; if(buf[j] > max) max = buf[j]; - buffer[i * 2] = min; - buffer[i * 2 + 1] = max; + power += buf[j] * buf[j]; + } + + buffer[i * 3] = min; + buffer[i * 3 + 1] = max; + buffer[i * 3 + 2] = sqrt(power) / len; + + if(eos) + { + length = i; + break; } } diff --git a/intern/audaspace/intern/AUD_C-API.h b/intern/audaspace/intern/AUD_C-API.h index 7689d1b06b5..2cd24551dd9 100644 --- a/intern/audaspace/intern/AUD_C-API.h +++ b/intern/audaspace/intern/AUD_C-API.h @@ -502,7 +502,7 @@ extern void AUD_setSyncCallback(AUD_syncFunction function, void* data); extern int AUD_doesPlayback(void); -extern int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length); +extern int AUD_readSound(AUD_Sound* sound, sample_t* buffer, int length, int samples_per_second); extern AUD_Sound* AUD_copy(AUD_Sound* sound); diff --git a/release/scripts/startup/bl_ui/space_sequencer.py b/release/scripts/startup/bl_ui/space_sequencer.py index 5320297dce2..19202088faf 100644 --- a/release/scripts/startup/bl_ui/space_sequencer.py +++ b/release/scripts/startup/bl_ui/space_sequencer.py @@ -638,6 +638,7 @@ class SEQUENCER_PT_sound(SequencerButtonsPanel, bpy.types.Panel): row.prop(strip.sound, "use_memory_cache") + layout.prop(strip, "waveform") layout.prop(strip, "volume") layout.prop(strip, "attenuation") layout.prop(strip, "pitch") diff --git a/source/blender/blenkernel/BKE_sound.h b/source/blender/blenkernel/BKE_sound.h index c36532ee4cc..ecf0d7e459a 100644 --- a/source/blender/blenkernel/BKE_sound.h +++ b/source/blender/blenkernel/BKE_sound.h @@ -35,6 +35,8 @@ * \author nzc */ +#define SOUND_WAVE_SAMPLES_PER_SECOND 250 + struct PackedFile; struct bSound; struct bContext; @@ -42,6 +44,12 @@ struct ListBase; struct Main; struct Sequence; +typedef struct SoundWaveform +{ + int length; + float *data; +} SoundWaveform; + void sound_init_once(void); void sound_init(struct Main *main); @@ -122,7 +130,9 @@ float sound_sync_scene(struct Scene *scene); int sound_scene_playing(struct Scene *scene); -int sound_read_sound_buffer(struct bSound* sound, float* buffer, int length, float start, float end); +void sound_free_waveform(struct bSound* sound); + +void sound_read_waveform(struct bSound* sound); int sound_get_channels(struct bSound* sound); diff --git a/source/blender/blenkernel/intern/sound.c b/source/blender/blenkernel/intern/sound.c index 17df6ba23cb..888c719a304 100644 --- a/source/blender/blenkernel/intern/sound.c +++ b/source/blender/blenkernel/intern/sound.c @@ -24,6 +24,7 @@ #include "DNA_sound_types.h" #include "DNA_speaker_types.h" +#define WITH_AUDASPACE #ifdef WITH_AUDASPACE # include "AUD_C-API.h" #endif @@ -96,6 +97,8 @@ void sound_free(struct bSound* sound) AUD_unload(sound->cache); sound->cache = NULL; } + + sound_free_waveform(sound); #endif // WITH_AUDASPACE } @@ -608,12 +611,34 @@ int sound_scene_playing(struct Scene *scene) return -1; } -int sound_read_sound_buffer(struct bSound* sound, float* buffer, int length, float start, float end) +void sound_free_waveform(struct bSound* sound) +{ + if(sound->waveform) + { + MEM_freeN(((SoundWaveform*)sound->waveform)->data); + MEM_freeN(sound->waveform); + } + + sound->waveform = NULL; +} + +void sound_read_waveform(struct bSound* sound) { - AUD_Sound* limiter = AUD_limitSound(sound->cache, start, end); - int ret= AUD_readSound(limiter, buffer, length); - AUD_unload(limiter); - return ret; + AUD_SoundInfo info; + + info = AUD_getInfo(sound->playback_handle); + + if(info.length > 0) + { + SoundWaveform* waveform = MEM_mallocN(sizeof(SoundWaveform), "SoundWaveform");; + int length = info.length * SOUND_WAVE_SAMPLES_PER_SECOND; + + waveform->data = MEM_mallocN(length * sizeof(float) * 3, "SoundWaveform.samples"); + waveform->length = AUD_readSound(sound->playback_handle, waveform->data, length, SOUND_WAVE_SAMPLES_PER_SECOND); + + sound_free_waveform(sound); + sound->waveform = waveform; + } } int sound_get_channels(struct bSound* sound) diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c index ef71df31df4..ae5bafa2d08 100644 --- a/source/blender/blenloader/intern/readfile.c +++ b/source/blender/blenloader/intern/readfile.c @@ -5609,6 +5609,7 @@ static void direct_link_sound(FileData *fd, bSound *sound) { sound->handle = NULL; sound->playback_handle = NULL; + sound->waveform = NULL; // versioning stuff, if there was a cache, then we enable caching: if(sound->cache) @@ -11766,6 +11767,36 @@ static void do_versions(FileData *fd, Library *lib, Main *main) SEQ_END } } + { + bScreen *screen; + for(screen= main->screen.first; screen; screen= screen->id.next) { + ScrArea *sa; + /* add regions */ + for(sa= screen->areabase.first; sa; sa= sa->next) { + SpaceLink *sl= sa->spacedata.first; + if(sl->spacetype==SPACE_SEQ) { + ARegion *ar; + for (ar=sa->regionbase.first; ar; ar= ar->next) { + if(ar->regiontype == RGN_TYPE_WINDOW) { + if(ar->v2d.min[1] == 4.0f) + ar->v2d.min[1]= 0.5f; + } + } + } + for (sl= sa->spacedata.first; sl; sl= sl->next) { + if(sl->spacetype==SPACE_SEQ) { + ARegion *ar; + for (ar=sl->regionbase.first; ar; ar= ar->next) { + if(ar->regiontype == RGN_TYPE_WINDOW) { + if(ar->v2d.min[1] == 4.0f) + ar->v2d.min[1]= 0.5f; + } + } + } + } + } + } + } { /* Make "auto-clamped" handles a per-keyframe setting instead of per-FCurve * diff --git a/source/blender/editors/space_sequencer/sequencer_draw.c b/source/blender/editors/space_sequencer/sequencer_draw.c index 7e4207c75fe..bc97ff341db 100644 --- a/source/blender/editors/space_sequencer/sequencer_draw.c +++ b/source/blender/editors/space_sequencer/sequencer_draw.c @@ -173,30 +173,63 @@ static void drawseqwave(Scene *scene, Sequence *seq, float x1, float y1, float x x2 the end x value, same for y1 and y2 stepsize is width of a pixel. */ - if(seq->sound->cache) + if(seq->flag & SEQ_AUDIO_DRAW_WAVEFORM) { - int i; + int i, j, pos; int length = floor((x2-x1)/stepsize)+1; float ymid = (y1+y2)/2; float yscale = (y2-y1)/2; - float* samples = MEM_mallocN(length * sizeof(float) * 2, "seqwave_samples"); - if(!samples) - return; - if(sound_read_sound_buffer(seq->sound, samples, length, - (seq->startofs + seq->anim_startofs)/FPS, - (seq->startofs + seq->anim_startofs + seq->enddisp - seq->startdisp)/FPS) != length) + float samplestep; + float startsample, endsample; + float value; + + SoundWaveform* waveform; + + if(!seq->sound->waveform) + sound_read_waveform(seq->sound); + + waveform = seq->sound->waveform; + + startsample = floor((seq->startofs + seq->anim_startofs)/FPS * SOUND_WAVE_SAMPLES_PER_SECOND); + endsample = ceil((seq->startofs + seq->anim_startofs + seq->enddisp - seq->startdisp)/FPS * SOUND_WAVE_SAMPLES_PER_SECOND); + samplestep = (endsample-startsample) * stepsize / (x2-x1); + + if(length > floor((waveform->length - startsample) / samplestep)) + length = floor((waveform->length - startsample) / samplestep); + + glBegin(GL_LINE_STRIP); + for(i = 0; i < length; i++) { - MEM_freeN(samples); - return; + pos = startsample + i * samplestep; + + value = waveform->data[pos * 3]; + + for(j = pos+1; (j < waveform->length) && (j < pos + samplestep); j++) + { + if(value > waveform->data[j * 3]) + value = waveform->data[j * 3]; + } + + glVertex2f(x1+i*stepsize, ymid + value * yscale); } - glBegin(GL_LINES); + glEnd(); + + glBegin(GL_LINE_STRIP); for(i = 0; i < length; i++) { - glVertex2f(x1+i*stepsize, ymid + samples[i * 2] * yscale); - glVertex2f(x1+i*stepsize, ymid + samples[i * 2 + 1] * yscale); + pos = startsample + i * samplestep; + + value = waveform->data[pos * 3 + 1]; + + for(j = pos+1; (j < waveform->length) && (j < pos + samplestep); j++) + { + if(value < waveform->data[j * 3 + 1]) + value = waveform->data[j * 3 + 1]; + } + + glVertex2f(x1+i*stepsize, ymid + value * yscale); } glEnd(); - MEM_freeN(samples); } } diff --git a/source/blender/editors/space_sequencer/space_sequencer.c b/source/blender/editors/space_sequencer/space_sequencer.c index d1df9699fa3..36471c7ffcf 100644 --- a/source/blender/editors/space_sequencer/space_sequencer.c +++ b/source/blender/editors/space_sequencer/space_sequencer.c @@ -223,7 +223,7 @@ static SpaceLink *sequencer_new(const bContext *C) ar->v2d.cur= ar->v2d.tot; ar->v2d.min[0]= 10.0f; - ar->v2d.min[1]= 4.0f; + ar->v2d.min[1]= 0.5f; ar->v2d.max[0]= MAXFRAMEF; ar->v2d.max[1]= MAXSEQ; diff --git a/source/blender/makesdna/DNA_sequence_types.h b/source/blender/makesdna/DNA_sequence_types.h index aa04dc9ac13..359ef8449e9 100644 --- a/source/blender/makesdna/DNA_sequence_types.h +++ b/source/blender/makesdna/DNA_sequence_types.h @@ -288,6 +288,7 @@ typedef struct SpeedControlVars { #define SEQ_AUDIO_VOLUME_ANIMATED (1<<24) #define SEQ_AUDIO_PITCH_ANIMATED (1<<25) #define SEQ_AUDIO_PAN_ANIMATED (1<<26) +#define SEQ_AUDIO_DRAW_WAVEFORM (1<<27) #define SEQ_INVALID_EFFECT (1<<31) diff --git a/source/blender/makesdna/DNA_sound_types.h b/source/blender/makesdna/DNA_sound_types.h index 56ad0e1ff84..d7546e84bbf 100644 --- a/source/blender/makesdna/DNA_sound_types.h +++ b/source/blender/makesdna/DNA_sound_types.h @@ -84,6 +84,11 @@ typedef struct bSound { */ void *cache; + /** + * Waveform display data. + */ + void *waveform; + /** * The audaspace handle that should actually be played back. * Should be cache if cache != NULL; otherwise it's handle diff --git a/source/blender/makesrna/intern/rna_sequencer.c b/source/blender/makesrna/intern/rna_sequencer.c index 6e18f955bf5..38575242fd6 100644 --- a/source/blender/makesrna/intern/rna_sequencer.c +++ b/source/blender/makesrna/intern/rna_sequencer.c @@ -976,6 +976,11 @@ static void rna_def_sequence(BlenderRNA *brna) RNA_def_property_ui_text(prop, "Lock", "Lock strip so that it can't be transformed"); RNA_def_property_update(prop, NC_SCENE|ND_SEQUENCER, NULL); + prop= RNA_def_property(srna, "waveform", PROP_BOOLEAN, PROP_NONE); + RNA_def_property_boolean_sdna(prop, NULL, "flag", SEQ_AUDIO_DRAW_WAVEFORM); + RNA_def_property_ui_text(prop, "Draw Waveform", "Whether to draw the sound's waveform."); + RNA_def_property_update(prop, NC_SCENE|ND_SEQUENCER, NULL); + /* strip positioning */ prop= RNA_def_property(srna, "frame_final_duration", PROP_INT, PROP_TIME); -- cgit v1.2.3