diff options
Diffstat (limited to 'libspeex/jitter.c')
| -rw-r--r-- | libspeex/jitter.c | 862 |
1 files changed, 510 insertions, 352 deletions
diff --git a/libspeex/jitter.c b/libspeex/jitter.c index 2b64453..0bf609b 100644 --- a/libspeex/jitter.c +++ b/libspeex/jitter.c @@ -32,66 +32,260 @@ */ +/* +TODO: +- Add short-term estimate +- Defensive programming + + warn when last returned < last desired (begative buffering) + + warn if update_delay not called between get() and tick() or is called twice in a row +- Linked list structure for holding the packets instead of the current fixed-size array + + return memory to a pool + + allow pre-allocation of the pool + + optional max number of elements +- Statistics + + drift + + loss + + late + + jitter + + buffering delay +*/ #ifdef HAVE_CONFIG_H #include "config.h" #endif -#include "misc.h" +#include "arch.h" #include <speex/speex.h> #include <speex/speex_bits.h> #include <speex/speex_jitter.h> +#include "os_support.h" #ifndef NULL #define NULL 0 #endif -#define LATE_BINS 15 -#define MAX_MARGIN 30 /**< Number of bins in margin histogram */ - #define SPEEX_JITTER_MAX_BUFFER_SIZE 200 /**< Maximum number of packets in jitter buffer */ - +#define TSUB(a,b) ((spx_int32_t)((a)-(b))) #define GT32(a,b) (((spx_int32_t)((a)-(b)))>0) #define GE32(a,b) (((spx_int32_t)((a)-(b)))>=0) #define LT32(a,b) (((spx_int32_t)((a)-(b)))<0) #define LE32(a,b) (((spx_int32_t)((a)-(b)))<=0) +#define ROUND_DOWN(x, step) ((x)<0 ? ((x)-(step)+1)/(step)*(step) : (x)/(step)*(step)) + +#define MAX_TIMINGS 20 +#define MAX_BUFFERS 3 +#define TOP_DELAY 20 + +/** Buffer that keeps the time of arrival of the latest packets */ +struct TimingBuffer { + int filled; /**< Number of entries occupied in "timing" and "counts"*/ + int curr_count; /**< Number of packet timings we got (including those we discarded) */ + spx_int16_t timing[MAX_TIMINGS]; /**< Sorted list of all timings ("latest" packets first) */ + spx_int16_t counts[MAX_TIMINGS]; /**< Order the packets were put in (will be used for short-term estimate) */ +}; + +static void tb_init(struct TimingBuffer *tb) +{ + tb->filled = 0; + tb->curr_count = 0; +} + +/* Add the timing of a new packet to the TimingBuffer */ +static void tb_add(struct TimingBuffer *tb, spx_int16_t timing) +{ + int pos; + /* Discard packet that won't make it into the list because they're too early */ + if (tb->filled >= MAX_TIMINGS && timing >= tb->timing[tb->filled-1]) + { + tb->curr_count++; + return; + } + + /* Find where the timing info goes in the sorted list */ + pos = 0; + /* FIXME: Do bisection instead of linear search */ + while (pos<tb->filled && timing >= tb->timing[pos]) + { + pos++; + } + + speex_assert(pos <= tb->filled && pos < MAX_TIMINGS); + + /* Shift everything so we can perform the insertion */ + if (pos < tb->filled) + { + int move_size = tb->filled-pos; + if (tb->filled == MAX_TIMINGS) + move_size -= 1; + speex_move(&tb->timing[pos+1], &tb->timing[pos], move_size*sizeof(tb->timing[0])); + speex_move(&tb->counts[pos+1], &tb->counts[pos], move_size*sizeof(tb->counts[0])); + } + /* Insert */ + tb->timing[pos] = timing; + tb->counts[pos] = tb->curr_count; + + tb->curr_count++; + if (tb->filled<MAX_TIMINGS) + tb->filled++; +} + + + /** Jitter buffer structure */ struct JitterBuffer_ { - spx_uint32_t pointer_timestamp; /**< Timestamp of what we will *get* next */ - spx_uint32_t current_timestamp; /**< Timestamp of the local clock (what we will *play* next) */ - - char *buf[SPEEX_JITTER_MAX_BUFFER_SIZE]; /**< Buffer of packets (NULL if slot is free) */ - spx_uint32_t timestamp[SPEEX_JITTER_MAX_BUFFER_SIZE]; /**< Timestamp of packet */ - int span[SPEEX_JITTER_MAX_BUFFER_SIZE]; /**< Timestamp of packet */ - int len[SPEEX_JITTER_MAX_BUFFER_SIZE]; /**< Number of bytes in packet */ - - int tick_size; /**< Output granularity */ - int reset_state; /**< True if state was just reset */ - int buffer_margin; /**< How many frames we want to keep in the buffer (lower bound) */ - int late_cutoff; /**< How late must a packet be for it not to be considered at all */ - int interp_requested; /**< An interpolation is requested by speex_jitter_update_delay() */ - - int lost_count; /**< Number of consecutive lost packets */ - float shortterm_margin[MAX_MARGIN]; /**< Short term margin histogram */ - float longterm_margin[MAX_MARGIN]; /**< Long term margin histogram */ - float loss_rate; /**< Average loss rate */ + spx_uint32_t pointer_timestamp; /**< Timestamp of what we will *get* next */ + spx_uint32_t last_returned_timestamp; /**< Useful for getting the next packet with the same timestamp (for fragmented media) */ + spx_uint32_t next_stop; /**< Estimated time the next get() will be called */ + + spx_int32_t buffered; /**< Amount of data we think is still buffered by the application (timestamp units)*/ + + JitterBufferPacket packets[SPEEX_JITTER_MAX_BUFFER_SIZE]; /**< Packets stored in the buffer */ + spx_uint32_t arrival[SPEEX_JITTER_MAX_BUFFER_SIZE]; /**< Packet arrival time (0 means it was late, even though it's a valid timestamp) */ + + void (*destroy) (void *); /**< Callback for destroying a packet */ + + spx_int32_t delay_step; /**< Size of the steps when adjusting buffering (timestamp units) */ + spx_int32_t concealment_size; /**< Size of the packet loss concealment "units" */ + int reset_state; /**< True if state was just reset */ + int buffer_margin; /**< How many frames we want to keep in the buffer (lower bound) */ + int late_cutoff; /**< How late must a packet be for it not to be considered at all */ + int interp_requested; /**< An interpolation is requested by speex_jitter_update_delay() */ + + struct TimingBuffer _tb[MAX_BUFFERS]; /**< Don't use those directly */ + struct TimingBuffer *timeBuffers[MAX_BUFFERS]; /**< Storing arrival time of latest frames so we can compute some stats */ + int window_size; /**< Total window over which the late frames are counted */ + int subwindow_size; /**< Sub-window size for faster computation */ + int max_late_rate; /**< Absolute maximum amount of late packets tolerable (in percent) */ + int latency_tradeoff; /**< Latency equivalent of losing one percent of packets */ + int auto_tradeoff; /**< Latency equivalent of losing one percent of packets (automatic default) */ + + int lost_count; /**< Number of consecutive lost packets */ }; +/** Based on available data, this computes the optimal delay for the jitter buffer. + The optimised function is in timestamp units and is: + cost = delay + late_factor*[number of frames that would be late if we used that delay] + @param tb Array of buffers + @param late_factor Equivalent cost of a late frame (in timestamp units) + */ +static spx_int16_t compute_opt_delay(JitterBuffer *jitter) +{ + int i; + spx_int16_t opt=0; + spx_int32_t best_cost=0x7fffffff; + int late = 0; + int pos[MAX_BUFFERS]; + int tot_count; + float late_factor; + int penalty_taken = 0; + int best = 0; + int worst = 0; + spx_int32_t deltaT; + struct TimingBuffer *tb; + + tb = jitter->_tb; + + /* Number of packet timings we have received (including those we didn't keep) */ + tot_count = 0; + for (i=0;i<MAX_BUFFERS;i++) + tot_count += tb[i].curr_count; + if (tot_count==0) + return 0; + + /* Compute cost for one lost packet */ + if (jitter->latency_tradeoff != 0) + late_factor = jitter->latency_tradeoff * 100.0f / tot_count; + else + late_factor = jitter->auto_tradeoff * jitter->window_size/tot_count; + + /*fprintf(stderr, "late_factor = %f\n", late_factor);*/ + for (i=0;i<MAX_BUFFERS;i++) + pos[i] = 0; + + /* Pick the TOP_DELAY "latest" packets (doesn't need to actually be late + for the current settings) */ + for (i=0;i<TOP_DELAY;i++) + { + int j; + int next=-1; + int latest = 32767; + /* Pick latest amoung all sub-windows */ + for (j=0;j<MAX_BUFFERS;j++) + { + if (pos[j] < tb[j].filled && tb[j].timing[pos[j]] < latest) + { + next = j; + latest = tb[j].timing[pos[j]]; + } + } + if (next != -1) + { + spx_int32_t cost; + + if (i==0) + worst = latest; + best = latest; + latest = ROUND_DOWN(latest, jitter->delay_step); + pos[next]++; + + /* Actual cost function that tells us how bad using this delay would be */ + cost = -latest + late_factor*late; + /*fprintf(stderr, "cost %d = %d + %f * %d\n", cost, -latest, late_factor, late);*/ + if (cost < best_cost) + { + best_cost = cost; + opt = latest; + } + } else { + break; + } + + /* For the next timing we will consider, there will be one more late packet to count */ + late++; + /* Two-frame penalty if we're going to increase the amount of late frames (hysteresis) */ + if (latest >= 0 && !penalty_taken) + { + penalty_taken = 1; + late+=2; + } + } + + deltaT = best-worst; + /* This is a default "automatic latency tradeoff" when none is provided */ + jitter->auto_tradeoff = 1 + deltaT/TOP_DELAY; + /*fprintf(stderr, "auto_tradeoff = %d (%d %d %d)\n", jitter->auto_tradeoff, best, worst, i);*/ + + /* FIXME: Compute a short-term estimate too and combine with the long-term one */ + + /* Prevents reducing the buffer size when we haven't really had much data */ + if (tot_count < TOP_DELAY && opt > 0) + return 0; + return opt; +} + + /** Initialise jitter buffer */ -JitterBuffer *jitter_buffer_init(int tick) +JitterBuffer *jitter_buffer_init(void) { JitterBuffer *jitter = (JitterBuffer*)speex_alloc(sizeof(JitterBuffer)); if (jitter) { int i; + spx_int32_t tmp; for (i=0;i<SPEEX_JITTER_MAX_BUFFER_SIZE;i++) - jitter->buf[i]=NULL; - jitter->tick_size = tick; - jitter->buffer_margin = 1; + jitter->packets[i].data=NULL; + jitter->delay_step = 1; + jitter->concealment_size = 1; + /*FIXME: Should this be 0 or 1?*/ + jitter->buffer_margin = 0; jitter->late_cutoff = 50; + jitter->destroy = NULL; + jitter->latency_tradeoff = 0; + tmp = 4; + jitter_buffer_ctl(jitter, JITTER_BUFFER_SET_MAX_LATE_RATE, &tmp); jitter_buffer_reset(jitter); } return jitter; @@ -103,22 +297,27 @@ void jitter_buffer_reset(JitterBuffer *jitter) int i; for (i=0;i<SPEEX_JITTER_MAX_BUFFER_SIZE;i++) { - if (jitter->buf[i]) + if (jitter->packets[i].data) { - speex_free(jitter->buf[i]); - jitter->buf[i] = NULL; + if (jitter->destroy) + jitter->destroy(jitter->packets[i].data); + else + speex_free(jitter->packets[i].data); + jitter->packets[i].data = NULL; } } /* Timestamp is actually undefined at this point */ jitter->pointer_timestamp = 0; - jitter->current_timestamp = 0; + jitter->next_stop = 0; jitter->reset_state = 1; jitter->lost_count = 0; - jitter->loss_rate = 0; - for (i=0;i<MAX_MARGIN;i++) + jitter->buffered = 0; + jitter->auto_tradeoff = 32000; + + for (i=0;i<MAX_BUFFERS;i++) { - jitter->shortterm_margin[i] = 0; - jitter->longterm_margin[i] = 0; + tb_init(&jitter->_tb[i]); + jitter->timeBuffers[i] = &jitter->_tb[i]; } /*fprintf (stderr, "reset\n");*/ } @@ -130,17 +329,52 @@ void jitter_buffer_destroy(JitterBuffer *jitter) speex_free(jitter); } +/** Take the following timing into consideration for future calculations */ +static void update_timings(JitterBuffer *jitter, spx_int32_t timing) +{ + if (timing < -32767) + timing = -32767; + if (timing > 32767) + timing = 32767; + /* If the current sub-window is full, perform a rotation and discard oldest sub-widow */ + if (jitter->timeBuffers[0]->curr_count >= jitter->subwindow_size) + { + int i; + /*fprintf(stderr, "Rotate buffer\n");*/ + struct TimingBuffer *tmp = jitter->timeBuffers[MAX_BUFFERS-1]; + for (i=MAX_BUFFERS-1;i>=1;i--) + jitter->timeBuffers[i] = jitter->timeBuffers[i-1]; + jitter->timeBuffers[0] = tmp; + tb_init(jitter->timeBuffers[0]); + } + tb_add(jitter->timeBuffers[0], timing); +} + +/** Compensate all timings when we do an adjustment of the buffering */ +static void shift_timings(JitterBuffer *jitter, spx_int16_t amount) +{ + int i, j; + for (i=0;i<MAX_BUFFERS;i++) + { + for (j=0;j<jitter->timeBuffers[i]->filled;j++) + jitter->timeBuffers[i]->timing[j] += amount; + } +} + + /** Put one packet into the jitter buffer */ void jitter_buffer_put(JitterBuffer *jitter, const JitterBufferPacket *packet) { int i,j; - spx_int32_t arrival_margin; + int late; /*fprintf (stderr, "put packet %d %d\n", timestamp, span);*/ + + /* Syncing on the first packet to arrive */ if (jitter->reset_state) { jitter->reset_state=0; jitter->pointer_timestamp = packet->timestamp; - jitter->current_timestamp = packet->timestamp; + jitter->next_stop = packet->timestamp; /*fprintf(stderr, "reset to %d\n", timestamp);*/ } @@ -148,167 +382,120 @@ void jitter_buffer_put(JitterBuffer *jitter, const JitterBufferPacket *packet) for (i=0;i<SPEEX_JITTER_MAX_BUFFER_SIZE;i++) { /* Make sure we don't discard a "just-late" packet in case we want to play it next (if we interpolate). */ - if (jitter->buf[i] && LE32(jitter->timestamp[i] + jitter->span[i], jitter->pointer_timestamp)) + if (jitter->packets[i].data && LE32(jitter->packets[i].timestamp + jitter->packets[i].span, jitter->pointer_timestamp)) { /*fprintf (stderr, "cleaned (not played)\n");*/ - speex_free(jitter->buf[i]); - jitter->buf[i] = NULL; + if (jitter->destroy) + jitter->destroy(jitter->packets[i].data); + else + speex_free(jitter->packets[i].data); + jitter->packets[i].data = NULL; } } - /*Find an empty slot in the buffer*/ - for (i=0;i<SPEEX_JITTER_MAX_BUFFER_SIZE;i++) + /*fprintf(stderr, "arrival: %d %d %d\n", packet->timestamp, jitter->next_stop, jitter->pointer_timestamp);*/ + /* Check if packet is late (could still be useful though) */ + if (LT32(packet->timestamp, jitter->next_stop)) { - if (jitter->buf[i]==NULL) - break; + update_timings(jitter, ((spx_int32_t)packet->timestamp) - ((spx_int32_t)jitter->next_stop) - jitter->buffer_margin); + late = 1; + } else { + late = 0; } - - /*fprintf(stderr, "%d %d %f\n", timestamp, jitter->pointer_timestamp, jitter->drift_average);*/ - /*No place left in the buffer*/ - if (i==SPEEX_JITTER_MAX_BUFFER_SIZE) + + /* Only insert the packet if it's not hopelessly late (i.e. totally useless) */ + if (GE32(packet->timestamp+packet->span+jitter->delay_step, jitter->pointer_timestamp)) { - int earliest=jitter->timestamp[0]; - i=0; - for (j=1;j<SPEEX_JITTER_MAX_BUFFER_SIZE;j++) + + /*Find an empty slot in the buffer*/ + for (i=0;i<SPEEX_JITTER_MAX_BUFFER_SIZE;i++) { - if (!jitter->buf[i] || LT32(jitter->timestamp[j],earliest)) + if (jitter->packets[i].data==NULL) + break; + } + + /*No place left in the buffer, need to make room for it by discarding the oldest packet */ + if (i==SPEEX_JITTER_MAX_BUFFER_SIZE) + { + int earliest=jitter->packets[0].timestamp; + i=0; + for (j=1;j<SPEEX_JITTER_MAX_BUFFER_SIZE;j++) { - earliest = jitter->timestamp[j]; - i=j; + if (!jitter->packets[i].data || LT32(jitter->packets[j].timestamp,earliest)) + { + earliest = jitter->packets[j].timestamp; + i=j; + } } + if (jitter->destroy) + jitter->destroy(jitter->packets[i].data); + else + speex_free(jitter->packets[i].data); + jitter->packets[i].data=NULL; + if (jitter->lost_count>20) + { + jitter_buffer_reset(jitter); + } + /*fprintf (stderr, "Buffer is full, discarding earliest frame %d (currently at %d)\n", timestamp, jitter->pointer_timestamp);*/ } - speex_free(jitter->buf[i]); - jitter->buf[i]=NULL; - if (jitter->lost_count>20) + + /* Copy packet in buffer */ + if (jitter->destroy) { - jitter_buffer_reset(jitter); + jitter->packets[i].data = packet->data; + } else { + jitter->packets[i].data=(char*)speex_alloc(packet->len); + for (j=0;j<packet->len;j++) + jitter->packets[i].data[j]=packet->data[j]; } - /*fprintf (stderr, "Buffer is full, discarding earliest frame %d (currently at %d)\n", timestamp, jitter->pointer_timestamp);*/ + jitter->packets[i].timestamp=packet->timestamp; + jitter->packets[i].span=packet->span; + jitter->packets[i].len=packet->len; + jitter->packets[i].user_data=packet->user_data; + if (late) + jitter->arrival[i] = 0; + else + jitter->arrival[i] = jitter->next_stop; } - /* Copy packet in buffer */ - jitter->buf[i]=(char*)speex_alloc(packet->len); - for (j=0;j<packet->len;j++) - jitter->buf[i][j]=packet->data[j]; - jitter->timestamp[i]=packet->timestamp; - jitter->span[i]=packet->span; - jitter->len[i]=packet->len; - - /* Adjust the buffer size depending on network conditions. - The arrival margin is how much in advance (or late) the packet it */ - arrival_margin = (((spx_int32_t)packet->timestamp) - ((spx_int32_t)jitter->current_timestamp))/jitter->tick_size - jitter->buffer_margin; - if (arrival_margin >= -jitter->late_cutoff) - { - /* Here we compute the histogram based on the time of arrival of the packet. - This is based on a (first-order) recursive average. We keep both a short-term - histogram and a long-term histogram */ - spx_int32_t int_margin; - /* First, apply the "damping" of the recursive average to all bins */ - for (i=0;i<MAX_MARGIN;i++) - { - jitter->shortterm_margin[i] *= .98; - jitter->longterm_margin[i] *= .995; - } - /* What histogram bin the packet should be counted in */ - int_margin = LATE_BINS + arrival_margin; - if (int_margin>MAX_MARGIN-1) - int_margin = MAX_MARGIN-1; - if (int_margin<0) - int_margin = 0; - /* Add the packet to the right bin */ - jitter->shortterm_margin[int_margin] += .02; - jitter->longterm_margin[int_margin] += .005; - } else { - /* Packet has arrived *way* too late, we pretty much consider it lost and not take it into account in the histogram */ - /*fprintf (stderr, "way too late = %d\n", arrival_margin);*/ - if (jitter->lost_count>20) - { - jitter_buffer_reset(jitter); - } - } -#if 0 /* Enable to check how much is being buffered */ - if (rand()%1000==0) - { - int count = 0; - for (j=0;j<SPEEX_JITTER_MAX_BUFFER_SIZE;j++) - { - if (jitter->buf[j]) - count++; - } - fprintf (stderr, "buffer_size = %d\n", count); - } -#endif } /** Get one packet from the jitter buffer */ -int jitter_buffer_get(JitterBuffer *jitter, JitterBufferPacket *packet, spx_int32_t *start_offset) +int jitter_buffer_get(JitterBuffer *jitter, JitterBufferPacket *packet, spx_int32_t desired_span, spx_int32_t *start_offset) { int i; unsigned int j; - float late_ratio_short; - float late_ratio_long; - float ontime_ratio_short; - float ontime_ratio_long; - float early_ratio_short; - float early_ratio_long; - int chunk_size; int incomplete = 0; + spx_int16_t opt; - if (jitter->interp_requested) + jitter->last_returned_timestamp = jitter->pointer_timestamp; + + if (jitter->interp_requested != 0) { - jitter->interp_requested = 0; if (start_offset) *start_offset = 0; packet->timestamp = jitter->pointer_timestamp; - packet->span = jitter->tick_size; - jitter->pointer_timestamp += jitter->tick_size; + packet->span = jitter->interp_requested; + + /* Increment the pointer because it got decremented in the delay update */ + jitter->pointer_timestamp += jitter->interp_requested; packet->len = 0; - return JITTER_BUFFER_MISSING; - } - if (LT32(jitter->current_timestamp+jitter->tick_size, jitter->pointer_timestamp)) - { - jitter->current_timestamp = jitter->pointer_timestamp; - speex_warning("did you forget to call jitter_buffer_tick() by any chance?"); - } - /*fprintf (stderr, "get packet %d %d\n", jitter->pointer_timestamp, jitter->current_timestamp);*/ + /*fprintf (stderr, "Deferred interpolate\n");*/ + + jitter->interp_requested = 0; + + jitter->buffered = packet->span - desired_span; - /* FIXME: This should be only what remaining of the current tick */ - chunk_size = jitter->tick_size; - - /* Compiling arrival statistics */ - - late_ratio_short = 0; - late_ratio_long = 0; - /* Count the proportion of packets that are late */ - for (i=0;i<LATE_BINS;i++) - { - late_ratio_short += jitter->shortterm_margin[i]; - late_ratio_long += jitter->longterm_margin[i]; - } - /* Count the proportion of packets that are just on time */ - ontime_ratio_short = jitter->shortterm_margin[LATE_BINS]; - ontime_ratio_long = jitter->longterm_margin[LATE_BINS]; - early_ratio_short = early_ratio_long = 0; - /* Count the proportion of packets that are early */ - for (i=LATE_BINS+1;i<MAX_MARGIN;i++) - { - early_ratio_short += jitter->shortterm_margin[i]; - early_ratio_long += jitter->longterm_margin[i]; - } - if (0&&jitter->pointer_timestamp%1000==0) - { - /*fprintf (stderr, "%f %f %f %f %f %f\n", early_ratio_short, early_ratio_long, ontime_ratio_short, ontime_ratio_long, late_ratio_short, late_ratio_long);*/ - /*fprintf (stderr, "%f %f\n", early_ratio_short + ontime_ratio_short + late_ratio_short, early_ratio_long + ontime_ratio_long + late_ratio_long);*/ + return JITTER_BUFFER_MISSING; } - /* Searching for the packet that fits best */ /* Search the buffer for a packet with the right timestamp and spanning the whole current chunk */ for (i=0;i<SPEEX_JITTER_MAX_BUFFER_SIZE;i++) { - if (jitter->buf[i] && jitter->timestamp[i]==jitter->pointer_timestamp && GE32(jitter->timestamp[i]+jitter->span[i],jitter->pointer_timestamp+chunk_size)) + if (jitter->packets[i].data && jitter->packets[i].timestamp==jitter->pointer_timestamp && GE32(jitter->packets[i].timestamp+jitter->packets[i].span,jitter->pointer_timestamp+desired_span)) break; } @@ -317,7 +504,7 @@ int jitter_buffer_get(JitterBuffer *jitter, JitterBufferPacket *packet, spx_int3 { for (i=0;i<SPEEX_JITTER_MAX_BUFFER_SIZE;i++) { - if (jitter->buf[i] && LE32(jitter->timestamp[i], jitter->pointer_timestamp) && GE32(jitter->timestamp[i]+jitter->span[i],jitter->pointer_timestamp+chunk_size)) + if (jitter->packets[i].data && LE32(jitter->packets[i].timestamp, jitter->pointer_timestamp) && GE32(jitter->packets[i].timestamp+jitter->packets[i].span,jitter->pointer_timestamp+desired_span)) break; } } @@ -327,7 +514,7 @@ int jitter_buffer_get(JitterBuffer *jitter, JitterBufferPacket *packet, spx_int3 { for (i=0;i<SPEEX_JITTER_MAX_BUFFER_SIZE;i++) { - if (jitter->buf[i] && LE32(jitter->timestamp[i], jitter->pointer_timestamp) && GT32(jitter->timestamp[i]+jitter->span[i],jitter->pointer_timestamp)) + if (jitter->packets[i].data && LE32(jitter->packets[i].timestamp, jitter->pointer_timestamp) && GT32(jitter->packets[i].timestamp+jitter->packets[i].span,jitter->pointer_timestamp)) break; } } @@ -342,12 +529,12 @@ int jitter_buffer_get(JitterBuffer *jitter, JitterBufferPacket *packet, spx_int3 for (i=0;i<SPEEX_JITTER_MAX_BUFFER_SIZE;i++) { /* check if packet starts within current chunk */ - if (jitter->buf[i] && LT32(jitter->timestamp[i],jitter->pointer_timestamp+chunk_size) && GE32(jitter->timestamp[i],jitter->pointer_timestamp)) + if (jitter->packets[i].data && LT32(jitter->packets[i].timestamp,jitter->pointer_timestamp+desired_span) && GE32(jitter->packets[i].timestamp,jitter->pointer_timestamp)) { - if (!found || LT32(jitter->timestamp[i],best_time) || (jitter->timestamp[i]==best_time && GT32(jitter->span[i],best_span))) + if (!found || LT32(jitter->packets[i].timestamp,best_time) || (jitter->packets[i].timestamp==best_time && GT32(jitter->packets[i].span,best_span))) { - best_time = jitter->timestamp[i]; - best_span = jitter->span[i]; + best_time = jitter->packets[i].timestamp; + best_span = jitter->packets[i].span; besti = i; found = 1; } @@ -357,31 +544,52 @@ int jitter_buffer_get(JitterBuffer *jitter, JitterBufferPacket *packet, spx_int3 { i=besti; incomplete = 1; - /*fprintf (stderr, "incomplete: %d %d %d %d\n", jitter->timestamp[i], jitter->pointer_timestamp, chunk_size, jitter->span[i]);*/ + /*fprintf (stderr, "incomplete: %d %d %d %d\n", jitter->packets[i].timestamp, jitter->pointer_timestamp, chunk_size, jitter->packets[i].span);*/ } } /* If we find something */ if (i!=SPEEX_JITTER_MAX_BUFFER_SIZE) { + + /* We (obviously) haven't lost this packet */ jitter->lost_count = 0; - jitter->loss_rate = .999*jitter->loss_rate; - /* Check for potential overflow */ - packet->len = jitter->len[i]; + + /* In this case, 0 isn't as a valid timestamp */ + if (jitter->arrival[i] != 0) + { + update_timings(jitter, ((spx_int32_t)jitter->packets[i].timestamp) - ((spx_int32_t)jitter->arrival[i]) - jitter->buffer_margin); + } + + + /* FIXME: Check for potential overflow */ + packet->len = jitter->packets[i].len; /* Copy packet */ - for (j=0;j<packet->len;j++) - packet->data[j] = jitter->buf[i][j]; - /* Remove packet */ - speex_free(jitter->buf[i]); - jitter->buf[i] = NULL; + if (jitter->destroy) + { + packet->data = jitter->packets[i].data; + } else { + for (j=0;j<packet->len;j++) + packet->data[j] = jitter->packets[i].data[j]; + /* Remove packet */ + speex_free(jitter->packets[i].data); + } + jitter->packets[i].data = NULL; /* Set timestamp and span (if requested) */ if (start_offset) - *start_offset = (spx_int32_t)jitter->timestamp[i]-(spx_int32_t)jitter->pointer_timestamp; - packet->timestamp = jitter->timestamp[i]; - packet->span = jitter->span[i]; - /* Point at the end of the current packet */ - jitter->pointer_timestamp = jitter->timestamp[i]+jitter->span[i]; + *start_offset = (spx_int32_t)jitter->packets[i].timestamp-(spx_int32_t)jitter->pointer_timestamp; + + packet->timestamp = jitter->packets[i].timestamp; + jitter->last_returned_timestamp = packet->timestamp; + + packet->span = jitter->packets[i].span; + packet->user_data = jitter->packets[i].user_data; + /* Point to the end of the current packet */ + jitter->pointer_timestamp = jitter->packets[i].timestamp+jitter->packets[i].span; + + jitter->buffered = *start_offset + packet->span - desired_span; + if (incomplete) return JITTER_BUFFER_INCOMPLETE; else @@ -390,41 +598,81 @@ int jitter_buffer_get(JitterBuffer *jitter, JitterBufferPacket *packet, spx_int3 /* If we haven't found anything worth returning */ + /*fprintf (stderr, "not found\n");*/ jitter->lost_count++; /*fprintf (stderr, "m");*/ /*fprintf (stderr, "lost_count = %d\n", jitter->lost_count);*/ - jitter->loss_rate = .999*jitter->loss_rate + .001; if (start_offset) *start_offset = 0; - packet->timestamp = jitter->pointer_timestamp; - packet->span = jitter->tick_size; - jitter->pointer_timestamp += chunk_size; - packet->len = 0; - /* Adjusting the buffering bssed on the amount of packets that are early/on time/late */ - if (late_ratio_short > .1 || late_ratio_long > .03) + opt = compute_opt_delay(jitter); + + /* Should we force an increase in the buffer or just do normal interpolation? */ + if (opt < 0) { - /* If too many packets are arriving late */ - jitter->shortterm_margin[MAX_MARGIN-1] += jitter->shortterm_margin[MAX_MARGIN-2]; - jitter->longterm_margin[MAX_MARGIN-1] += jitter->longterm_margin[MAX_MARGIN-2]; - for (i=MAX_MARGIN-3;i>=0;i--) - { - jitter->shortterm_margin[i+1] = jitter->shortterm_margin[i]; - jitter->longterm_margin[i+1] = jitter->longterm_margin[i]; - } - jitter->shortterm_margin[0] = 0; - jitter->longterm_margin[0] = 0; - jitter->pointer_timestamp -= jitter->tick_size; - jitter->current_timestamp -= jitter->tick_size; - /*fprintf (stderr, "i");*/ - /*fprintf (stderr, "interpolate (getting some slack)\n");*/ + /* Need to increase buffering */ + + /* Shift histogram to compensate */ + shift_timings(jitter, -opt); + + packet->timestamp = jitter->pointer_timestamp; + packet->span = -opt; + /* Don't move the pointer_timestamp forward */ + packet->len = 0; + + /*jitter->pointer_timestamp -= jitter->delay_step;*/ + /*fprintf (stderr, "Forced to interpolate\n");*/ + } else { + /* Normal packet loss */ + packet->timestamp = jitter->pointer_timestamp; + + desired_span = ROUND_DOWN(desired_span, jitter->concealment_size); + packet->span = desired_span; + jitter->pointer_timestamp += desired_span; + packet->len = 0; + /*fprintf (stderr, "Normal loss\n");*/ } + jitter->buffered = packet->span - desired_span; return JITTER_BUFFER_MISSING; } +int jitter_buffer_get_another(JitterBuffer *jitter, JitterBufferPacket *packet) +{ + int i, j; + for (i=0;i<SPEEX_JITTER_MAX_BUFFER_SIZE;i++) + { + if (jitter->packets[i].data && jitter->packets[i].timestamp==jitter->last_returned_timestamp) + break; + } + if (i!=SPEEX_JITTER_MAX_BUFFER_SIZE) + { + /* Copy packet */ + packet->len = jitter->packets[i].len; + if (jitter->destroy) + { + packet->data = jitter->packets[i].data; + } else { + for (j=0;j<packet->len;j++) + packet->data[j] = jitter->packets[i].data[j]; + /* Remove packet */ + speex_free(jitter->packets[i].data); + } + jitter->packets[i].data = NULL; + packet->timestamp = jitter->packets[i].timestamp; + packet->span = jitter->packets[i].span; + packet->user_data = jitter->packets[i].user_data; + return JITTER_BUFFER_OK; + } else { + packet->data = NULL; + packet->len = 0; + packet->span = 0; + return JITTER_BUFFER_MISSING; + } +} + /** Get pointer timestamp of jitter buffer */ int jitter_buffer_get_pointer_timestamp(JitterBuffer *jitter) { @@ -433,85 +681,44 @@ int jitter_buffer_get_pointer_timestamp(JitterBuffer *jitter) void jitter_buffer_tick(JitterBuffer *jitter) { - jitter->current_timestamp += jitter->tick_size; + if (jitter->buffered >= 0) + { + jitter->next_stop = jitter->pointer_timestamp - jitter->buffered; + } else { + jitter->next_stop = jitter->pointer_timestamp; + speex_warning_int("jitter buffer sees negative buffering, you code might be broken. Value is ", jitter->buffered); + } + jitter->buffered = 0; +} + +void jitter_buffer_remaining_span(JitterBuffer *jitter, spx_uint32_t rem) +{ + if (jitter->buffered < 0) + speex_warning_int("jitter buffer sees negative buffering, you code might be broken. Value is ", jitter->buffered); + jitter->next_stop = jitter->pointer_timestamp - rem; } /* Let the jitter buffer know it's the right time to adjust the buffering delay to the network conditions */ int jitter_buffer_update_delay(JitterBuffer *jitter, JitterBufferPacket *packet, spx_int32_t *start_offset) { - int i; - float late_ratio_short; - float late_ratio_long; - float ontime_ratio_short; - float ontime_ratio_long; - float early_ratio_short; - float early_ratio_long; - - if (LT32(jitter->current_timestamp+jitter->tick_size, jitter->pointer_timestamp)) - { - jitter->current_timestamp = jitter->pointer_timestamp; - speex_warning("did you forget to call jitter_buffer_tick() by any chance?"); - } - /*fprintf (stderr, "get packet %d %d\n", jitter->pointer_timestamp, jitter->current_timestamp);*/ - - /* FIXME: This should be only what remaining of the current tick */ - late_ratio_short = 0; - late_ratio_long = 0; - /* Count the proportion of packets that are late */ - for (i=0;i<LATE_BINS;i++) - { - late_ratio_short += jitter->shortterm_margin[i]; - late_ratio_long += jitter->longterm_margin[i]; - } - /* Count the proportion of packets that are just on time */ - ontime_ratio_short = jitter->shortterm_margin[LATE_BINS]; - ontime_ratio_long = jitter->longterm_margin[LATE_BINS]; - early_ratio_short = early_ratio_long = 0; - /* Count the proportion of packets that are early */ - for (i=LATE_BINS+1;i<MAX_MARGIN;i++) - { - early_ratio_short += jitter->shortterm_margin[i]; - early_ratio_long += jitter->longterm_margin[i]; - } + spx_int16_t opt = compute_opt_delay(jitter); + /*fprintf(stderr, "opt adjustment is %d ", opt);*/ - /* Adjusting the buffering bssed on the amount of packets that are early/on time/late */ - if (late_ratio_short > .1 || late_ratio_long > .03) + if (opt < 0) { - /* If too many packets are arriving late */ - jitter->shortterm_margin[MAX_MARGIN-1] += jitter->shortterm_margin[MAX_MARGIN-2]; - jitter->longterm_margin[MAX_MARGIN-1] += jitter->longterm_margin[MAX_MARGIN-2]; - for (i=MAX_MARGIN-3;i>=0;i--) - { - jitter->shortterm_margin[i+1] = jitter->shortterm_margin[i]; - jitter->longterm_margin[i+1] = jitter->longterm_margin[i]; - } - jitter->shortterm_margin[0] = 0; - jitter->longterm_margin[0] = 0; - jitter->pointer_timestamp -= jitter->tick_size; - jitter->current_timestamp -= jitter->tick_size; - jitter->interp_requested = 1; - return JITTER_BUFFER_ADJUST_INTERPOLATE; - - } else if (late_ratio_short + ontime_ratio_short < .005 && late_ratio_long + ontime_ratio_long < .01 && early_ratio_short > .8) + shift_timings(jitter, -opt); + + jitter->pointer_timestamp += opt; + jitter->interp_requested = -opt; + /*fprintf (stderr, "Decision to interpolate %d samples\n", -opt);*/ + } else if (opt > 0) { - /* Many frames arriving early */ - jitter->shortterm_margin[0] += jitter->shortterm_margin[1]; - jitter->longterm_margin[0] += jitter->longterm_margin[1]; - for (i=1;i<MAX_MARGIN-1;i++) - { - jitter->shortterm_margin[i] = jitter->shortterm_margin[i+1]; - jitter->longterm_margin[i] = jitter->longterm_margin[i+1]; - } - jitter->shortterm_margin[MAX_MARGIN-1] = 0; - jitter->longterm_margin[MAX_MARGIN-1] = 0; - /*fprintf (stderr, "drop frame\n");*/ - /*fprintf (stderr, "d");*/ - jitter->pointer_timestamp += jitter->tick_size; - jitter->current_timestamp += jitter->tick_size; - return JITTER_BUFFER_ADJUST_DROP; + shift_timings(jitter, -opt); + jitter->pointer_timestamp += opt; + /*fprintf (stderr, "Decision to drop %d samples\n", opt);*/ } - return JITTER_BUFFER_ADJUST_OK; + return opt; } /* Used like the ioctl function to control the jitter buffer parameters */ @@ -530,13 +737,45 @@ int jitter_buffer_ctl(JitterBuffer *jitter, int request, void *ptr) count = 0; for (i=0;i<SPEEX_JITTER_MAX_BUFFER_SIZE;i++) { - if (jitter->buf[i] && LE32(jitter->pointer_timestamp, jitter->timestamp[i])) + if (jitter->packets[i].data && LE32(jitter->pointer_timestamp, jitter->packets[i].timestamp)) { count++; } } *(spx_int32_t*)ptr = count; break; + case JITTER_BUFFER_SET_DESTROY_CALLBACK: + jitter->destroy = (void (*) (void *))ptr; + break; + case JITTER_BUFFER_GET_DESTROY_CALLBACK: + *(void (**) (void *))ptr = jitter->destroy; + break; + case JITTER_BUFFER_SET_DELAY_STEP: + jitter->delay_step = *(spx_int32_t*)ptr; + break; + case JITTER_BUFFER_GET_DELAY_STEP: + *(spx_int32_t*)ptr = jitter->delay_step; + break; + case JITTER_BUFFER_SET_CONCEALMENT_SIZE: + jitter->concealment_size = *(spx_int32_t*)ptr; + break; + case JITTER_BUFFER_GET_CONCEALMENT_SIZE: + *(spx_int32_t*)ptr = jitter->concealment_size; + break; + case JITTER_BUFFER_SET_MAX_LATE_RATE: + jitter->max_late_rate = *(spx_int32_t*)ptr; + jitter->window_size = 100*TOP_DELAY/jitter->max_late_rate; + jitter->subwindow_size = jitter->window_size/MAX_BUFFERS; + break; + case JITTER_BUFFER_GET_MAX_LATE_RATE: + *(spx_int32_t*)ptr = jitter->max_late_rate; + break; + case JITTER_BUFFER_SET_LATE_COST: + jitter->latency_tradeoff = *(spx_int32_t*)ptr; + break; + case JITTER_BUFFER_GET_LATE_COST: + *(spx_int32_t*)ptr = jitter->latency_tradeoff; + break; default: speex_warning_int("Unknown jitter_buffer_ctl request: ", request); return -1; @@ -544,84 +783,3 @@ int jitter_buffer_ctl(JitterBuffer *jitter, int request, void *ptr) return 0; } - - -void speex_jitter_init(SpeexJitter *jitter, void *decoder, int sampling_rate) -{ - jitter->dec = decoder; - speex_decoder_ctl(decoder, SPEEX_GET_FRAME_SIZE, &jitter->frame_size); - - jitter->packets = jitter_buffer_init(jitter->frame_size); - - speex_bits_init(&jitter->current_packet); - jitter->valid_bits = 0; - -} - -void speex_jitter_destroy(SpeexJitter *jitter) -{ - jitter_buffer_destroy(jitter->packets); - speex_bits_destroy(&jitter->current_packet); -} - -void speex_jitter_put(SpeexJitter *jitter, char *packet, int len, int timestamp) -{ - JitterBufferPacket p; - p.data = packet; - p.len = len; - p.timestamp = timestamp; - p.span = jitter->frame_size; - jitter_buffer_put(jitter->packets, &p); -} - -void speex_jitter_get(SpeexJitter *jitter, short *out, int *current_timestamp) -{ - int i; - int ret; - char data[2048]; - JitterBufferPacket packet; - packet.data = data; - - if (jitter->valid_bits) - { - /* Try decoding last received packet */ - ret = speex_decode_int(jitter->dec, &jitter->current_packet, out); - if (ret == 0) - { - jitter_buffer_tick(jitter->packets); - return; - } else { - jitter->valid_bits = 0; - } - } - - ret = jitter_buffer_get(jitter->packets, &packet, NULL); - - if (ret != JITTER_BUFFER_OK) - { - /* No packet found */ - - /*fprintf (stderr, "lost/late frame\n");*/ - /*Packet is late or lost*/ - speex_decode_int(jitter->dec, NULL, out); - } else { - speex_bits_read_from(&jitter->current_packet, packet.data, packet.len); - /* Decode packet */ - ret = speex_decode_int(jitter->dec, &jitter->current_packet, out); - if (ret == 0) - { - jitter->valid_bits = 1; - } else { - /* Error while decoding */ - for (i=0;i<jitter->frame_size;i++) - out[i]=0; - } - } - jitter_buffer_update_delay(jitter->packets, &packet, NULL); - jitter_buffer_tick(jitter->packets); -} - -int speex_jitter_get_pointer_timestamp(SpeexJitter *jitter) -{ - return jitter_buffer_get_pointer_timestamp(jitter->packets); -} |