diff options
28 files changed, 3530 insertions, 813 deletions
diff --git a/extern/libmv/CMakeLists.txt b/extern/libmv/CMakeLists.txt index cf0ad1102e0..60cd84d89d4 100644 --- a/extern/libmv/CMakeLists.txt +++ b/extern/libmv/CMakeLists.txt @@ -62,6 +62,7 @@ set(SRC libmv/multiview/fundamental.cc libmv/multiview/projection.cc libmv/multiview/triangulation.cc + libmv/multiview/homography.cc libmv/numeric/numeric.cc libmv/numeric/poly.cc libmv/simple_pipeline/bundle.cc @@ -84,6 +85,7 @@ set(SRC libmv/tracking/pyramid_region_tracker.cc libmv/tracking/retrack_region_tracker.cc libmv/tracking/trklt_region_tracker.cc + libmv/tracking/track_region.cc third_party/fast/fast_10.c third_party/fast/fast_11.c diff --git a/extern/libmv/libmv-capi.cpp b/extern/libmv/libmv-capi.cpp index 6c20d76eeac..ffa065f08fe 100644 --- a/extern/libmv/libmv-capi.cpp +++ b/extern/libmv/libmv-capi.cpp @@ -28,6 +28,10 @@ tracking between which failed */ #undef DUMP_FAILURE +/* define this to generate PNG images with content of search areas + on every itteration of tracking */ +#undef DUMP_ALWAYS + #include "libmv-capi.h" #include "third_party/gflags/gflags/gflags.h" @@ -45,6 +49,7 @@ #include "libmv/tracking/trklt_region_tracker.h" #include "libmv/tracking/lmicklt_region_tracker.h" #include "libmv/tracking/pyramid_region_tracker.h" +#include "libmv/tracking/track_region.h" #include "libmv/simple_pipeline/callbacks.h" #include "libmv/simple_pipeline/tracks.h" @@ -59,7 +64,7 @@ #include <stdlib.h> #include <assert.h> -#ifdef DUMP_FAILURE +#if defined(DUMP_FAILURE) || defined (DUMP_ALWAYS) # include <png.h> #endif @@ -97,7 +102,7 @@ void libmv_initLogging(const char *argv0) void libmv_startDebugLogging(void) { google::SetCommandLineOption("logtostderr", "1"); - google::SetCommandLineOption("v", "0"); + google::SetCommandLineOption("v", "2"); google::SetCommandLineOption("stderrthreshold", "1"); google::SetCommandLineOption("minloglevel", "0"); V3D::optimizerVerbosenessLevel = 1; @@ -158,20 +163,35 @@ libmv_RegionTracker *libmv_bruteRegionTrackerNew(int half_window_size, double mi return (libmv_RegionTracker *)brute_region_tracker; } -static void floatBufToImage(const float *buf, int width, int height, libmv::FloatImage *image) +static void floatBufToImage(const float *buf, int width, int height, int channels, libmv::FloatImage *image) { - int x, y, a = 0; + int x, y, k, a = 0; - image->resize(height, width); + image->Resize(height, width, channels); for (y = 0; y < height; y++) { for (x = 0; x < width; x++) { - (*image)(y, x, 0) = buf[a++]; + for (k = 0; k < channels; k++) { + (*image)(y, x, k) = buf[a++]; + } + } + } +} + +static void imageToFloatBuf(const libmv::FloatImage *image, int channels, float *buf) +{ + int x, y, k, a = 0; + + for (y = 0; y < image->Height(); y++) { + for (x = 0; x < image->Width(); x++) { + for (k = 0; k < channels; k++) { + buf[a++] = (*image)(y, x, k); + } } } } -#ifdef DUMP_FAILURE +#if defined(DUMP_FAILURE) || defined (DUMP_ALWAYS) void savePNGImage(png_bytep *row_pointers, int width, int height, int depth, int color_type, char *file_name) { png_infop info_ptr; @@ -234,14 +254,14 @@ static void saveImage(char *prefix, libmv::FloatImage image, int x0, int y0) row_pointers[y]= (png_bytep)malloc(sizeof(png_byte)*4*image.Width()); for (x = 0; x < image.Width(); x++) { - if (x0 == x && y0 == y) { + if (x0 == x && image.Height() - y0 - 1 == y) { row_pointers[y][x*4+0]= 255; row_pointers[y][x*4+1]= 0; row_pointers[y][x*4+2]= 0; row_pointers[y][x*4+3]= 255; } else { - float pixel = image(y, x, 0); + float pixel = image(image.Height() - y - 1, x, 0); row_pointers[y][x*4+0]= pixel*255; row_pointers[y][x*4+1]= pixel*255; row_pointers[y][x*4+2]= pixel*255; @@ -302,19 +322,23 @@ int libmv_regionTrackerTrack(libmv_RegionTracker *libmv_tracker, const float *im libmv::RegionTracker *region_tracker = (libmv::RegionTracker *)libmv_tracker; libmv::FloatImage old_patch, new_patch; - floatBufToImage(ima1, width, height, &old_patch); - floatBufToImage(ima2, width, height, &new_patch); + floatBufToImage(ima1, width, height, 1, &old_patch); + floatBufToImage(ima2, width, height, 1, &new_patch); -#ifndef DUMP_FAILURE +#if !defined(DUMP_FAILURE) && !defined(DUMP_ALWAYS) return region_tracker->Track(old_patch, new_patch, x1, y1, x2, y2); #else { - double sx2 = *x2, sy2 = *y2; + /* double sx2 = *x2, sy2 = *y2; */ int result = region_tracker->Track(old_patch, new_patch, x1, y1, x2, y2); +#if defined(DUMP_ALWAYS) + { +#else if (!result) { +#endif saveImage("old_patch", old_patch, x1, y1); - saveImage("new_patch", new_patch, sx2, sy2); + saveImage("new_patch", new_patch, *x2, *y2); } return result; @@ -329,6 +353,103 @@ void libmv_regionTrackerDestroy(libmv_RegionTracker *libmv_tracker) delete region_tracker; } +/* ************ Planar tracker ************ */ + +/* TrackRegion (new planar tracker) */ +int libmv_trackRegion(const struct libmv_trackRegionOptions *options, + const float *image1, const float *image2, + int width, int height, + const double *x1, const double *y1, + struct libmv_trackRegionResult *result, + double *x2, double *y2) +{ + double xx1[5], yy1[5]; + double xx2[5], yy2[5]; + bool tracking_result = false; + + /* Convert to doubles for the libmv api. The four corners and the center. */ + for (int i = 0; i < 5; ++i) { + xx1[i] = x1[i]; + yy1[i] = y1[i]; + xx2[i] = x2[i]; + yy2[i] = y2[i]; + } + + libmv::TrackRegionOptions track_region_options; + switch (options->motion_model) { +#define LIBMV_CONVERT(the_model) \ + case libmv::TrackRegionOptions::the_model: \ + track_region_options.mode = libmv::TrackRegionOptions::the_model; \ + break; + LIBMV_CONVERT(TRANSLATION) + LIBMV_CONVERT(TRANSLATION_ROTATION) + LIBMV_CONVERT(TRANSLATION_SCALE) + LIBMV_CONVERT(TRANSLATION_ROTATION_SCALE) + LIBMV_CONVERT(AFFINE) + LIBMV_CONVERT(HOMOGRAPHY) +#undef LIBMV_CONVERT + } + + track_region_options.minimum_correlation = options->minimum_correlation; + track_region_options.max_iterations = options->num_iterations; + track_region_options.sigma = options->sigma; + track_region_options.num_extra_points = 1; + track_region_options.image1_mask = NULL; + track_region_options.use_brute_initialization = options->use_brute; + track_region_options.use_normalized_intensities = options->use_normalization; + + /* Convert from raw float buffers to libmv's FloatImage. */ + libmv::FloatImage old_patch, new_patch; + floatBufToImage(image1, width, height, 1, &old_patch); + floatBufToImage(image2, width, height, 1, &new_patch); + + libmv::TrackRegionResult track_region_result; + libmv::TrackRegion(old_patch, new_patch, xx1, yy1, track_region_options, xx2, yy2, &track_region_result); + + /* Convert to floats for the blender api. */ + for (int i = 0; i < 5; ++i) { + x2[i] = xx2[i]; + y2[i] = yy2[i]; + } + + /* TODO(keir): Update the termination string with failure details. */ + if (track_region_result.termination == libmv::TrackRegionResult::PARAMETER_TOLERANCE || + track_region_result.termination == libmv::TrackRegionResult::FUNCTION_TOLERANCE || + track_region_result.termination == libmv::TrackRegionResult::GRADIENT_TOLERANCE || + track_region_result.termination == libmv::TrackRegionResult::NO_CONVERGENCE) + { + tracking_result = true; + } + +#if defined(DUMP_FAILURE) || defined(DUMP_ALWAYS) +#if defined(DUMP_ALWAYS) + { +#else + if (!tracking_result) { +#endif + saveImage("old_patch", old_patch, x1[4], y1[4]); + saveImage("new_patch", new_patch, x2[4], y2[4]); + } +#endif + + return tracking_result; +} + +void libmv_samplePlanarPatch(const float *image, int width, int height, + int channels, const double *xs, const double *ys, + int num_samples_x, int num_samples_y, float *patch, + double *warped_position_x, double *warped_position_y) +{ + libmv::FloatImage libmv_image, libmv_patch; + + floatBufToImage(image, width, height, channels, &libmv_image); + + libmv::SamplePlanarPatch(libmv_image, xs, ys, num_samples_x, num_samples_y, + &libmv_patch, warped_position_x, warped_position_y); + + imageToFloatBuf(&libmv_patch, channels, patch); +} + /* ************ Tracks ************ */ libmv_Tracks *libmv_tracksNew(void) diff --git a/extern/libmv/libmv-capi.h b/extern/libmv/libmv-capi.h index bccc4706832..6f4b5dea384 100644 --- a/extern/libmv/libmv-capi.h +++ b/extern/libmv/libmv-capi.h @@ -50,6 +50,32 @@ int libmv_regionTrackerTrack(struct libmv_RegionTracker *libmv_tracker, const fl int width, int height, double x1, double y1, double *x2, double *y2); void libmv_regionTrackerDestroy(struct libmv_RegionTracker *libmv_tracker); +/* TrackRegion (new planar tracker) */ +struct libmv_trackRegionOptions { + int motion_model; + int num_iterations; + int use_brute; + int use_normalization; + double minimum_correlation; + double sigma; +}; +struct libmv_trackRegionResult { + int termination; + const char *termination_reason; + double correlation; +}; +int libmv_trackRegion(const struct libmv_trackRegionOptions *options, + const float *image1, const float *image2, + int width, int height, + const double *x1, const double *y1, + struct libmv_trackRegionResult *result, + double *x2, double *y2); + +void libmv_samplePlanarPatch(const float *image, int width, int height, + int channels, const double *xs, const double *ys, + int num_samples_x, int num_samples_y, float *patch, + double *warped_position_x, double *warped_position_y); + /* Tracks */ struct libmv_Tracks *libmv_tracksNew(void); void libmv_tracksInsert(struct libmv_Tracks *libmv_tracks, int image, int track, double x, double y); diff --git a/extern/libmv/libmv/image/sample.h b/extern/libmv/libmv/image/sample.h index e842747e6d4..a8850effeab 100644 --- a/extern/libmv/libmv/image/sample.h +++ b/extern/libmv/libmv/image/sample.h @@ -34,25 +34,22 @@ inline T SampleNearest(const Array3D<T> &image, return image(i, j, v); } -static inline void LinearInitAxis(float fx, int width, - int *x1, int *x2, - float *dx1, float *dx2) { - const int ix = int(fx); +inline void LinearInitAxis(float x, int size, + int *x1, int *x2, + float *dx) { + const int ix = static_cast<int>(x); if (ix < 0) { *x1 = 0; *x2 = 0; - *dx1 = 1; - *dx2 = 0; - } else if (ix > width-2) { - *x1 = width-1; - *x2 = width-1; - *dx1 = 1; - *dx2 = 0; + *dx = 1.0; + } else if (ix > size - 2) { + *x1 = size - 1; + *x2 = size - 1; + *dx = 1.0; } else { *x1 = ix; - *x2 = *x1 + 1; - *dx1 = *x2 - fx; - *dx2 = 1 - *dx1; + *x2 = ix + 1; + *dx = *x2 - x; } } @@ -60,18 +57,47 @@ static inline void LinearInitAxis(float fx, int width, template<typename T> inline T SampleLinear(const Array3D<T> &image, float y, float x, int v = 0) { int x1, y1, x2, y2; - float dx1, dy1, dx2, dy2; + float dx, dy; - LinearInitAxis(y, image.Height(), &y1, &y2, &dy1, &dy2); - LinearInitAxis(x, image.Width(), &x1, &x2, &dx1, &dx2); + // Take the upper left corner as integer pixel positions. + x -= 0.5; + y -= 0.5; + + LinearInitAxis(y, image.Height(), &y1, &y2, &dy); + LinearInitAxis(x, image.Width(), &x1, &x2, &dx); const T im11 = image(y1, x1, v); const T im12 = image(y1, x2, v); const T im21 = image(y2, x1, v); const T im22 = image(y2, x2, v); - return T(dy1 * ( dx1 * im11 + dx2 * im12 ) + - dy2 * ( dx1 * im21 + dx2 * im22 )); + return T( dy * ( dx * im11 + (1.0 - dx) * im12 ) + + (1 - dy) * ( dx * im21 + (1.0 - dx) * im22 )); +} + +/// Linear interpolation, of all channels. The sample is assumed to have the +/// same size as the number of channels in image. +template<typename T> +inline void SampleLinear(const Array3D<T> &image, float y, float x, T *sample) { + int x1, y1, x2, y2; + float dx, dy; + + // Take the upper left corner as integer pixel positions. + x -= 0.5; + y -= 0.5; + + LinearInitAxis(y, image.Height(), &y1, &y2, &dy); + LinearInitAxis(x, image.Width(), &x1, &x2, &dx); + + for (int i = 0; i < image.Depth(); ++i) { + const T im11 = image(y1, x1, i); + const T im12 = image(y1, x2, i); + const T im21 = image(y2, x1, i); + const T im22 = image(y2, x2, i); + + sample[i] = T( dy * ( dx * im11 + (1.0 - dx) * im12 ) + + (1 - dy) * ( dx * im21 + (1.0 - dx) * im22 )); + } } // Downsample all channels by 2. If the image has odd width or height, the last diff --git a/extern/libmv/libmv/multiview/homography.cc b/extern/libmv/libmv/multiview/homography.cc new file mode 100644 index 00000000000..366392f3923 --- /dev/null +++ b/extern/libmv/libmv/multiview/homography.cc @@ -0,0 +1,267 @@ +// Copyright (c) 2008, 2009 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. + +#include "libmv/logging/logging.h" +#include "libmv/multiview/homography.h" +#include "libmv/multiview/homography_parameterization.h" + +namespace libmv { +/** 2D Homography transformation estimation in the case that points are in + * euclidean coordinates. + * + * x = H y + * x and y vector must have the same direction, we could write + * crossproduct(|x|, * H * |y| ) = |0| + * + * | 0 -1 x2| |a b c| |y1| |0| + * | 1 0 -x1| * |d e f| * |y2| = |0| + * |-x2 x1 0| |g h 1| |1 | |0| + * + * That gives : + * + * (-d+x2*g)*y1 + (-e+x2*h)*y2 + -f+x2 |0| + * (a-x1*g)*y1 + (b-x1*h)*y2 + c-x1 = |0| + * (-x2*a+x1*d)*y1 + (-x2*b+x1*e)*y2 + -x2*c+x1*f |0| + */ +bool Homography2DFromCorrespondencesLinearEuc( + const Mat &x1, + const Mat &x2, + Mat3 *H, + double expected_precision) { + assert(2 == x1.rows()); + assert(4 <= x1.cols()); + assert(x1.rows() == x2.rows()); + assert(x1.cols() == x2.cols()); + + int n = x1.cols(); + MatX8 L = Mat::Zero(n * 3, 8); + Mat b = Mat::Zero(n * 3, 1); + for (int i = 0; i < n; ++i) { + int j = 3 * i; + L(j, 0) = x1(0, i); // a + L(j, 1) = x1(1, i); // b + L(j, 2) = 1.0; // c + L(j, 6) = -x2(0, i) * x1(0, i); // g + L(j, 7) = -x2(0, i) * x1(1, i); // h + b(j, 0) = x2(0, i); // i + + ++j; + L(j, 3) = x1(0, i); // d + L(j, 4) = x1(1, i); // e + L(j, 5) = 1.0; // f + L(j, 6) = -x2(1, i) * x1(0, i); // g + L(j, 7) = -x2(1, i) * x1(1, i); // h + b(j, 0) = x2(1, i); // i + + // This ensures better stability + // TODO(julien) make a lite version without this 3rd set + ++j; + L(j, 0) = x2(1, i) * x1(0, i); // a + L(j, 1) = x2(1, i) * x1(1, i); // b + L(j, 2) = x2(1, i); // c + L(j, 3) = -x2(0, i) * x1(0, i); // d + L(j, 4) = -x2(0, i) * x1(1, i); // e + L(j, 5) = -x2(0, i) ; // f + } + // Solve Lx=B + Vec h = L.fullPivLu().solve(b); + Homography2DNormalizedParameterization<double>::To(h, H); + if ((L * h).isApprox(b, expected_precision)) { + return true; + } else { + return false; + } +} + +/** 2D Homography transformation estimation in the case that points are in + * homogeneous coordinates. + * + * | 0 -x3 x2| |a b c| |y1| -x3*d+x2*g -x3*e+x2*h -x3*f+x2*1 |y1| (-x3*d+x2*g)*y1 (-x3*e+x2*h)*y2 (-x3*f+x2*1)*y3 |0| + * | x3 0 -x1| * |d e f| * |y2| = x3*a-x1*g x3*b-x1*h x3*c-x1*1 * |y2| = (x3*a-x1*g)*y1 (x3*b-x1*h)*y2 (x3*c-x1*1)*y3 = |0| + * |-x2 x1 0| |g h 1| |y3| -x2*a+x1*d -x2*b+x1*e -x2*c+x1*f |y3| (-x2*a+x1*d)*y1 (-x2*b+x1*e)*y2 (-x2*c+x1*f)*y3 |0| + * X = |a b c d e f g h|^t + */ +bool Homography2DFromCorrespondencesLinear(const Mat &x1, + const Mat &x2, + Mat3 *H, + double expected_precision) { + if (x1.rows() == 2) { + return Homography2DFromCorrespondencesLinearEuc(x1, x2, H, + expected_precision); + } + assert(3 == x1.rows()); + assert(4 <= x1.cols()); + assert(x1.rows() == x2.rows()); + assert(x1.cols() == x2.cols()); + + const int x = 0; + const int y = 1; + const int w = 2; + int n = x1.cols(); + MatX8 L = Mat::Zero(n * 3, 8); + Mat b = Mat::Zero(n * 3, 1); + for (int i = 0; i < n; ++i) { + int j = 3 * i; + L(j, 0) = x2(w, i) * x1(x, i);//a + L(j, 1) = x2(w, i) * x1(y, i);//b + L(j, 2) = x2(w, i) * x1(w, i);//c + L(j, 6) = -x2(x, i) * x1(x, i);//g + L(j, 7) = -x2(x, i) * x1(y, i);//h + b(j, 0) = x2(x, i) * x1(w, i); + + ++j; + L(j, 3) = x2(w, i) * x1(x, i);//d + L(j, 4) = x2(w, i) * x1(y, i);//e + L(j, 5) = x2(w, i) * x1(w, i);//f + L(j, 6) = -x2(y, i) * x1(x, i);//g + L(j, 7) = -x2(y, i) * x1(y, i);//h + b(j, 0) = x2(y, i) * x1(w, i); + + // This ensures better stability + ++j; + L(j, 0) = x2(y, i) * x1(x, i);//a + L(j, 1) = x2(y, i) * x1(y, i);//b + L(j, 2) = x2(y, i) * x1(w, i);//c + L(j, 3) = -x2(x, i) * x1(x, i);//d + L(j, 4) = -x2(x, i) * x1(y, i);//e + L(j, 5) = -x2(x, i) * x1(w, i);//f + } + // Solve Lx=B + Vec h = L.fullPivLu().solve(b); + if ((L * h).isApprox(b, expected_precision)) { + Homography2DNormalizedParameterization<double>::To(h, H); + return true; + } else { + return false; + } +} +/** + * x2 ~ A * x1 + * x2^t * Hi * A *x1 = 0 + * H1 = H2 = H3 = + * | 0 0 0 1| |-x2w| |0 0 0 0| | 0 | | 0 0 1 0| |-x2z| + * | 0 0 0 0| -> | 0 | |0 0 1 0| -> |-x2z| | 0 0 0 0| -> | 0 | + * | 0 0 0 0| | 0 | |0-1 0 0| | x2y| |-1 0 0 0| | x2x| + * |-1 0 0 0| | x2x| |0 0 0 0| | 0 | | 0 0 0 0| | 0 | + * H4 = H5 = H6 = + * |0 0 0 0| | 0 | | 0 1 0 0| |-x2y| |0 0 0 0| | 0 | + * |0 0 0 1| -> |-x2w| |-1 0 0 0| -> | x2x| |0 0 0 0| -> | 0 | + * |0 0 0 0| | 0 | | 0 0 0 0| | 0 | |0 0 0 1| |-x2w| + * |0-1 0 0| | x2y| | 0 0 0 0| | 0 | |0 0-1 0| | x2z| + * |a b c d| + * A = |e f g h| + * |i j k l| + * |m n o 1| + * + * x2^t * H1 * A *x1 = (-x2w*a +x2x*m )*x1x + (-x2w*b +x2x*n )*x1y + (-x2w*c +x2x*o )*x1z + (-x2w*d +x2x*1 )*x1w = 0 + * x2^t * H2 * A *x1 = (-x2z*e +x2y*i )*x1x + (-x2z*f +x2y*j )*x1y + (-x2z*g +x2y*k )*x1z + (-x2z*h +x2y*l )*x1w = 0 + * x2^t * H3 * A *x1 = (-x2z*a +x2x*i )*x1x + (-x2z*b +x2x*j )*x1y + (-x2z*c +x2x*k )*x1z + (-x2z*d +x2x*l )*x1w = 0 + * x2^t * H4 * A *x1 = (-x2w*e +x2y*m )*x1x + (-x2w*f +x2y*n )*x1y + (-x2w*g +x2y*o )*x1z + (-x2w*h +x2y*1 )*x1w = 0 + * x2^t * H5 * A *x1 = (-x2y*a +x2x*e )*x1x + (-x2y*b +x2x*f )*x1y + (-x2y*c +x2x*g )*x1z + (-x2y*d +x2x*h )*x1w = 0 + * x2^t * H6 * A *x1 = (-x2w*i +x2z*m )*x1x + (-x2w*j +x2z*n )*x1y + (-x2w*k +x2z*o )*x1z + (-x2w*l +x2z*1 )*x1w = 0 + * + * X = |a b c d e f g h i j k l m n o|^t +*/ +bool Homography3DFromCorrespondencesLinear(const Mat &x1, + const Mat &x2, + Mat4 *H, + double expected_precision) { + assert(4 == x1.rows()); + assert(5 <= x1.cols()); + assert(x1.rows() == x2.rows()); + assert(x1.cols() == x2.cols()); + const int x = 0; + const int y = 1; + const int z = 2; + const int w = 3; + int n = x1.cols(); + MatX15 L = Mat::Zero(n * 6, 15); + Mat b = Mat::Zero(n * 6, 1); + for (int i = 0; i < n; ++i) { + int j = 6 * i; + L(j, 0) = -x2(w, i) * x1(x, i);//a + L(j, 1) = -x2(w, i) * x1(y, i);//b + L(j, 2) = -x2(w, i) * x1(z, i);//c + L(j, 3) = -x2(w, i) * x1(w, i);//d + L(j,12) = x2(x, i) * x1(x, i);//m + L(j,13) = x2(x, i) * x1(y, i);//n + L(j,14) = x2(x, i) * x1(z, i);//o + b(j, 0) = -x2(x, i) * x1(w, i); + + ++j; + L(j, 4) = -x2(z, i) * x1(x, i);//e + L(j, 5) = -x2(z, i) * x1(y, i);//f + L(j, 6) = -x2(z, i) * x1(z, i);//g + L(j, 7) = -x2(z, i) * x1(w, i);//h + L(j, 8) = x2(y, i) * x1(x, i);//i + L(j, 9) = x2(y, i) * x1(y, i);//j + L(j,10) = x2(y, i) * x1(z, i);//k + L(j,11) = x2(y, i) * x1(w, i);//l + + ++j; + L(j, 0) = -x2(z, i) * x1(x, i);//a + L(j, 1) = -x2(z, i) * x1(y, i);//b + L(j, 2) = -x2(z, i) * x1(z, i);//c + L(j, 3) = -x2(z, i) * x1(w, i);//d + L(j, 8) = x2(x, i) * x1(x, i);//i + L(j, 9) = x2(x, i) * x1(y, i);//j + L(j,10) = x2(x, i) * x1(z, i);//k + L(j,11) = x2(x, i) * x1(w, i);//l + + ++j; + L(j, 4) = -x2(w, i) * x1(x, i);//e + L(j, 5) = -x2(w, i) * x1(y, i);//f + L(j, 6) = -x2(w, i) * x1(z, i);//g + L(j, 7) = -x2(w, i) * x1(w, i);//h + L(j,12) = x2(y, i) * x1(x, i);//m + L(j,13) = x2(y, i) * x1(y, i);//n + L(j,14) = x2(y, i) * x1(z, i);//o + b(j, 0) = -x2(y, i) * x1(w, i); + + ++j; + L(j, 0) = -x2(y, i) * x1(x, i);//a + L(j, 1) = -x2(y, i) * x1(y, i);//b + L(j, 2) = -x2(y, i) * x1(z, i);//c + L(j, 3) = -x2(y, i) * x1(w, i);//d + L(j, 4) = x2(x, i) * x1(x, i);//e + L(j, 5) = x2(x, i) * x1(y, i);//f + L(j, 6) = x2(x, i) * x1(z, i);//g + L(j, 7) = x2(x, i) * x1(w, i);//h + + ++j; + L(j, 8) = -x2(w, i) * x1(x, i);//i + L(j, 9) = -x2(w, i) * x1(y, i);//j + L(j,10) = -x2(w, i) * x1(z, i);//k + L(j,11) = -x2(w, i) * x1(w, i);//l + L(j,12) = x2(z, i) * x1(x, i);//m + L(j,13) = x2(z, i) * x1(y, i);//n + L(j,14) = x2(z, i) * x1(z, i);//o + b(j, 0) = -x2(z, i) * x1(w, i); + } + // Solve Lx=B + Vec h = L.fullPivLu().solve(b); + if ((L * h).isApprox(b, expected_precision)) { + Homography3DNormalizedParameterization<double>::To(h, H); + return true; + } else { + return false; + } +} +} // namespace libmv diff --git a/extern/libmv/libmv/multiview/homography.h b/extern/libmv/libmv/multiview/homography.h new file mode 100644 index 00000000000..786fd3df8ca --- /dev/null +++ b/extern/libmv/libmv/multiview/homography.h @@ -0,0 +1,84 @@ +// Copyright (c) 2011 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. + +#ifndef LIBMV_MULTIVIEW_HOMOGRAPHY_H_ +#define LIBMV_MULTIVIEW_HOMOGRAPHY_H_ + +#include "libmv/numeric/numeric.h" + +namespace libmv { + +/** + * 2D homography transformation estimation. + * + * This function estimates the homography transformation from a list of 2D + * correspondences which represents either: + * + * - 3D points on a plane, with a general moving camera. + * - 3D points with a rotating camera (pure rotation). + * - 3D points + different planar projections + * + * \param x1 The first 2xN or 3xN matrix of euclidean or homogeneous points. + * \param x2 The second 2xN or 3xN matrix of euclidean or homogeneous points. + * \param H The 3x3 homography transformation matrix (8 dof) such that + * x2 = H * x1 with |a b c| + * H = |d e f| + * |g h 1| + * \param expected_precision The expected precision in order for instance + * to accept almost homography matrices. + * + * \return True if the transformation estimation has succeeded. + * \note There must be at least 4 non-colinear points. + */ +bool Homography2DFromCorrespondencesLinear(const Mat &x1, + const Mat &x2, + Mat3 *H, + double expected_precision = + EigenDouble::dummy_precision()); + +/** + * 3D Homography transformation estimation. + * + * This function can be used in order to estimate the homography transformation + * from a list of 3D correspondences. + * + * \param[in] x1 The first 4xN matrix of homogeneous points + * \param[in] x2 The second 4xN matrix of homogeneous points + * \param[out] H The 4x4 homography transformation matrix (15 dof) such that + * x2 = H * x1 with |a b c d| + * H = |e f g h| + * |i j k l| + * |m n o 1| + * \param[in] expected_precision The expected precision in order for instance + * to accept almost homography matrices. + * + * \return true if the transformation estimation has succeeded + * + * \note Need at least 5 non coplanar points + * \note Points coordinates must be in homogeneous coordinates + */ +bool Homography3DFromCorrespondencesLinear(const Mat &x1, + const Mat &x2, + Mat4 *H, + double expected_precision = + EigenDouble::dummy_precision()); +} // namespace libmv + +#endif // LIBMV_MULTIVIEW_HOMOGRAPHY_H_ diff --git a/extern/libmv/libmv/multiview/homography_parameterization.h b/extern/libmv/libmv/multiview/homography_parameterization.h new file mode 100644 index 00000000000..b31642eea15 --- /dev/null +++ b/extern/libmv/libmv/multiview/homography_parameterization.h @@ -0,0 +1,91 @@ +// Copyright (c) 2011 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. + +#ifndef LIBMV_MULTIVIEW_HOMOGRAPHY_PARAMETERIZATION_H_ +#define LIBMV_MULTIVIEW_HOMOGRAPHY_PARAMETERIZATION_H_ + +#include "libmv/numeric/numeric.h" + +namespace libmv { + +/** A parameterization of the 2D homography matrix that uses 8 parameters so + * that the matrix is normalized (H(2,2) == 1). + * The homography matrix H is built from a list of 8 parameters (a, b,...g, h) + * as follows + * |a b c| + * H = |d e f| + * |g h 1| + */ +template<typename T = double> +class Homography2DNormalizedParameterization { + public: + typedef Eigen::Matrix<T, 8, 1> Parameters; // a, b, ... g, h + typedef Eigen::Matrix<T, 3, 3> Parameterized; // H + + /// Convert from the 8 parameters to a H matrix. + static void To(const Parameters &p, Parameterized *h) { + *h << p(0), p(1), p(2), + p(3), p(4), p(5), + p(6), p(7), 1.0; + } + + /// Convert from a H matrix to the 8 parameters. + static void From(const Parameterized &h, Parameters *p) { + *p << h(0, 0), h(0, 1), h(0, 2), + h(1, 0), h(1, 1), h(1, 2), + h(2, 0), h(2, 1); + } +}; + +/** A parameterization of the 2D homography matrix that uses 15 parameters so + * that the matrix is normalized (H(3,3) == 1). + * The homography matrix H is built from a list of 15 parameters (a, b,...n, o) + * as follows + * |a b c d| + * H = |e f g h| + * |i j k l| + * |m n o 1| + */ +template<typename T = double> +class Homography3DNormalizedParameterization { + public: + typedef Eigen::Matrix<T, 15, 1> Parameters; // a, b, ... n, o + typedef Eigen::Matrix<T, 4, 4> Parameterized; // H + + /// Convert from the 15 parameters to a H matrix. + static void To(const Parameters &p, Parameterized *h) { + *h << p(0), p(1), p(2), p(3), + p(4), p(5), p(6), p(7), + p(8), p(9), p(10), p(11), + p(12), p(13), p(14), 1.0; + } + + /// Convert from a H matrix to the 15 parameters. + static void From(const Parameterized &h, Parameters *p) { + *p << h(0, 0), h(0, 1), h(0, 2), h(0, 3), + h(1, 0), h(1, 1), h(1, 2), h(1, 3), + h(2, 0), h(2, 1), h(2, 2), h(2, 3), + h(3, 0), h(3, 1), h(3, 2); + } +}; + +} // namespace libmv + +#endif // LIBMV_MULTIVIEW_HOMOGRAPHY_PARAMETERIZATION_H_ diff --git a/extern/libmv/libmv/tracking/esm_region_tracker.cc b/extern/libmv/libmv/tracking/esm_region_tracker.cc index 221fa4d081b..a8dc46d439b 100644 --- a/extern/libmv/libmv/tracking/esm_region_tracker.cc +++ b/extern/libmv/libmv/tracking/esm_region_tracker.cc @@ -29,6 +29,7 @@ #include "libmv/image/correlation.h" #include "libmv/image/sample.h" #include "libmv/numeric/numeric.h" +#include "libmv/tracking/track_region.h" namespace libmv { @@ -72,6 +73,44 @@ bool EsmRegionTracker::Track(const FloatImage &image1, return false; } + // XXX + // TODO(keir): Delete the block between the XXX's once the planar tracker is + // integrated into blender. + // + // For now, to test, replace the ESM tracker with the Ceres tracker in + // translation mode. In the future, this should get removed and alloed to + // co-exist, since Ceres is not as fast as the ESM implementation since it + // specializes for translation. + double xx1[4], yy1[4]; + double xx2[4], yy2[4]; + // Clockwise winding, starting from the "origin" (top-left). + xx1[0] = xx2[0] = x1 - half_window_size; + yy1[0] = yy2[0] = y1 - half_window_size; + + xx1[1] = xx2[1] = x1 + half_window_size; + yy1[1] = yy2[1] = y1 - half_window_size; + + xx1[2] = xx2[2] = x1 + half_window_size; + yy1[2] = yy2[2] = y1 + half_window_size; + + xx1[3] = xx2[3] = x1 - half_window_size; + yy1[3] = yy2[3] = y1 + half_window_size; + + TrackRegionOptions options; + options.mode = TrackRegionOptions::TRANSLATION; + options.max_iterations = 20; + options.sigma = sigma; + options.use_esm = true; + + TrackRegionResult result; + TrackRegion(image1, image2, xx1, yy1, options, xx2, yy2, &result); + + *x2 = xx2[0] + half_window_size; + *y2 = yy2[0] + half_window_size; + + return true; + + // XXX int width = 2 * half_window_size + 1; // TODO(keir): Avoid recomputing gradients for e.g. the pyramid tracker. diff --git a/extern/libmv/libmv/tracking/track_region.cc b/extern/libmv/libmv/tracking/track_region.cc new file mode 100644 index 00000000000..e65ead50c80 --- /dev/null +++ b/extern/libmv/libmv/tracking/track_region.cc @@ -0,0 +1,1391 @@ +// Copyright (c) 2012 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@google.com (Keir Mierle) +// +// TODO(keir): While this tracking code works rather well, it has some +// outragous inefficiencies. There is probably a 5-10x speedup to be had if a +// smart coder went through the TODO's and made the suggested performance +// enhancements. + +// Necessary for M_E when building with MSVC. +#define _USE_MATH_DEFINES + +#include "libmv/tracking/track_region.h" + +#include <Eigen/SVD> +#include <Eigen/QR> +#include <iostream> +#include "ceres/ceres.h" +#include "libmv/logging/logging.h" +#include "libmv/image/image.h" +#include "libmv/image/sample.h" +#include "libmv/image/convolve.h" +#include "libmv/multiview/homography.h" +#include "libmv/numeric/numeric.h" + +namespace libmv { + +using ceres::Jet; +using ceres::JetOps; +using ceres::Chain; + +TrackRegionOptions::TrackRegionOptions() + : mode(TRANSLATION), + minimum_correlation(0), + max_iterations(20), + use_esm(true), + use_brute_initialization(true), + use_normalized_intensities(false), + sigma(0.9), + num_extra_points(0), + regularization_coefficient(0.0), + image1_mask(NULL) { +} + +namespace { + +// TODO(keir): Consider adding padding. +template<typename T> +bool InBounds(const FloatImage &image, + const T &x, + const T &y) { + return 0.0 <= x && x < image.Width() && + 0.0 <= y && y < image.Height(); +} + +bool AllInBounds(const FloatImage &image, + const double *x, + const double *y) { + for (int i = 0; i < 4; ++i) { + if (!InBounds(image, x[i], y[i])) { + return false; + } + } + return true; +} + +// Sample the image at position (x, y) but use the gradient, if present, to +// propagate derivatives from x and y. This is needed to integrate the numeric +// image gradients with Ceres's autodiff framework. +template<typename T> +static T SampleWithDerivative(const FloatImage &image_and_gradient, + const T &x, + const T &y) { + float scalar_x = JetOps<T>::GetScalar(x); + float scalar_y = JetOps<T>::GetScalar(y); + + // Note that sample[1] and sample[2] will be uninitialized in the scalar + // case, but that is not an issue because the Chain::Rule below will not read + // the uninitialized values. + float sample[3]; + if (JetOps<T>::IsScalar()) { + // For the scalar case, only sample the image. + sample[0] = SampleLinear(image_and_gradient, scalar_y, scalar_x, 0); + } else { + // For the derivative case, sample the gradient as well. + SampleLinear(image_and_gradient, scalar_y, scalar_x, sample); + } + T xy[2] = { x, y }; + return Chain<float, 2, T>::Rule(sample[0], sample + 1, xy); +} + +template<typename Warp> +class BoundaryCheckingCallback : public ceres::IterationCallback { + public: + BoundaryCheckingCallback(const FloatImage& image2, + const Warp &warp, + const double *x1, const double *y1) + : image2_(image2), warp_(warp), x1_(x1), y1_(y1) {} + + virtual ceres::CallbackReturnType operator()( + const ceres::IterationSummary& summary) { + // Warp the original 4 points with the current warp into image2. + double x2[4]; + double y2[4]; + for (int i = 0; i < 4; ++i) { + warp_.Forward(warp_.parameters, x1_[i], y1_[i], x2 + i, y2 + i); + } + // Enusre they are all in bounds. + if (!AllInBounds(image2_, x2, y2)) { + return ceres::SOLVER_ABORT; + } + return ceres::SOLVER_CONTINUE; + } + + private: + const FloatImage &image2_; + const Warp &warp_; + const double *x1_; + const double *y1_; +}; + +template<typename Warp> +class PixelDifferenceCostFunctor { + public: + PixelDifferenceCostFunctor(const TrackRegionOptions &options, + const FloatImage &image_and_gradient1, + const FloatImage &image_and_gradient2, + const Mat3 &canonical_to_image1, + int num_samples_x, + int num_samples_y, + const Warp &warp) + : options_(options), + image_and_gradient1_(image_and_gradient1), + image_and_gradient2_(image_and_gradient2), + canonical_to_image1_(canonical_to_image1), + num_samples_x_(num_samples_x), + num_samples_y_(num_samples_y), + warp_(warp), + pattern_and_gradient_(num_samples_y_, num_samples_x_, 3), + pattern_positions_(num_samples_y_, num_samples_x_, 2), + pattern_mask_(num_samples_y_, num_samples_x_, 1) { + ComputeCanonicalPatchAndNormalizer(); + } + + void ComputeCanonicalPatchAndNormalizer() { + src_mean_ = 0.0; + double num_samples = 0.0; + for (int r = 0; r < num_samples_y_; ++r) { + for (int c = 0; c < num_samples_x_; ++c) { + // Compute the position; cache it. + Vec3 image_position = canonical_to_image1_ * Vec3(c, r, 1); + image_position /= image_position(2); + pattern_positions_(r, c, 0) = image_position(0); + pattern_positions_(r, c, 1) = image_position(1); + + // Sample the pattern and gradients. + SampleLinear(image_and_gradient1_, + image_position(1), // SampleLinear is r, c. + image_position(0), + &pattern_and_gradient_(r, c, 0)); + + // Sample sample the mask. + double mask_value = 1.0; + if (options_.image1_mask != NULL) { + SampleLinear(*options_.image1_mask, + image_position(1), // SampleLinear is r, c. + image_position(0), + &pattern_mask_(r, c, 0)); + mask_value = pattern_mask_(r, c); + } + src_mean_ += pattern_and_gradient_(r, c, 0) * mask_value; + num_samples += mask_value; + } + } + src_mean_ /= num_samples; + } + + template<typename T> + bool operator()(const T *warp_parameters, T *residuals) const { + if (options_.image1_mask != NULL) { + VLOG(2) << "Using a mask."; + } + for (int i = 0; i < Warp::NUM_PARAMETERS; ++i) { + VLOG(2) << "warp_parameters[" << i << "]: " << warp_parameters[i]; + } + + T dst_mean = T(1.0); + if (options_.use_normalized_intensities) { + ComputeNormalizingCoefficient(warp_parameters, + &dst_mean); + } + + int cursor = 0; + for (int r = 0; r < num_samples_y_; ++r) { + for (int c = 0; c < num_samples_x_; ++c) { + // Use the pre-computed image1 position. + Vec2 image1_position(pattern_positions_(r, c, 0), + pattern_positions_(r, c, 1)); + + // Sample the mask early; if it's zero, this pixel has no effect. This + // allows early bailout from the expensive sampling that happens below. + // + // Note that partial masks are not short circuited. To see why short + // circuiting produces bitwise-exact same results, consider that the + // residual for each pixel is + // + // residual = mask * (src - dst) , + // + // and for jets, multiplying by a scalar multiplies the derivative + // components by the scalar as well. Therefore, if the mask is exactly + // zero, then so too will the final residual and derivatives. + double mask_value = 1.0; + if (options_.image1_mask != NULL) { + mask_value = pattern_mask_(r, c); + if (mask_value == 0.0) { + residuals[cursor++] = T(0.0); + continue; + } + } + + // Compute the location of the destination pixel. + T image2_position[2]; + warp_.Forward(warp_parameters, + T(image1_position[0]), + T(image1_position[1]), + &image2_position[0], + &image2_position[1]); + + // Sample the destination, propagating derivatives. + T dst_sample = SampleWithDerivative(image_and_gradient2_, + image2_position[0], + image2_position[1]); + + // Sample the source. This is made complicated by ESM mode. + T src_sample; + if (options_.use_esm && !JetOps<T>::IsScalar()) { + // In ESM mode, the derivative of the source is also taken into + // account. This changes the linearization in a way that causes + // better convergence. Copy the derivative of the warp parameters + // onto the jets for the image1 position. This is the ESM hack. + T image1_position_jet[2] = { + image2_position[0], // Order is x, y. This matches the + image2_position[1] // derivative order in the patch. + }; + JetOps<T>::SetScalar(image1_position[0], image1_position_jet + 0); + JetOps<T>::SetScalar(image1_position[1], image1_position_jet + 1); + + // Now that the image1 positions have the jets applied from the + // image2 position (the ESM hack), chain the image gradients to + // obtain a sample with the derivative with respect to the warp + // parameters attached. + src_sample = Chain<float, 2, T>::Rule(pattern_and_gradient_(r, c), + &pattern_and_gradient_(r, c, 1), + image1_position_jet); + + // The jacobians for these should be averaged. Due to the subtraction + // below, flip the sign of the src derivative so that the effect + // after subtraction of the jets is that they are averaged. + JetOps<T>::ScaleDerivative(-0.5, &src_sample); + JetOps<T>::ScaleDerivative(0.5, &dst_sample); + } else { + // This is the traditional, forward-mode KLT solution. + src_sample = T(pattern_and_gradient_(r, c)); + } + + // Normalize the samples by the mean values of each signal. The typical + // light model assumes multiplicative intensity changes with changing + // light, so this is a reasonable choice. Note that dst_mean has + // derivative information attached thanks to autodiff. + if (options_.use_normalized_intensities) { + src_sample /= T(src_mean_); + dst_sample /= dst_mean; + } + + // The difference is the error. + T error = src_sample - dst_sample; + + // Weight the error by the mask, if one is present. + if (options_.image1_mask != NULL) { + error *= T(mask_value); + } + residuals[cursor++] = error; + } + } + return true; + } + + // For normalized matching, the average and + template<typename T> + void ComputeNormalizingCoefficient(const T *warp_parameters, + T *dst_mean) const { + + *dst_mean = T(0.0); + double num_samples = 0.0; + for (int r = 0; r < num_samples_y_; ++r) { + for (int c = 0; c < num_samples_x_; ++c) { + // Use the pre-computed image1 position. + Vec2 image1_position(pattern_positions_(r, c, 0), + pattern_positions_(r, c, 1)); + + // Sample the mask early; if it's zero, this pixel has no effect. This + // allows early bailout from the expensive sampling that happens below. + double mask_value = 1.0; + if (options_.image1_mask != NULL) { + mask_value = pattern_mask_(r, c); + if (mask_value == 0.0) { + continue; + } + } + + // Compute the location of the destination pixel. + T image2_position[2]; + warp_.Forward(warp_parameters, + T(image1_position[0]), + T(image1_position[1]), + &image2_position[0], + &image2_position[1]); + + + // Sample the destination, propagating derivatives. + // TODO(keir): This accumulation can, surprisingly, be done as a + // pre-pass by using integral images. This is complicated by the need + // to store the jets in the integral image, but it is possible. + T dst_sample = SampleWithDerivative(image_and_gradient2_, + image2_position[0], + image2_position[1]); + + // Weight the sample by the mask, if one is present. + if (options_.image1_mask != NULL) { + dst_sample *= T(mask_value); + } + + *dst_mean += dst_sample; + num_samples += mask_value; + } + } + *dst_mean /= T(num_samples); + LG << "Normalization for dst:" << *dst_mean; + } + + // TODO(keir): Consider also computing the cost here. + double PearsonProductMomentCorrelationCoefficient( + const double *warp_parameters) const { + for (int i = 0; i < Warp::NUM_PARAMETERS; ++i) { + VLOG(2) << "Correlation warp_parameters[" << i << "]: " + << warp_parameters[i]; + } + + // The single-pass PMCC computation is somewhat numerically unstable, but + // it's sufficient for the tracker. + double sX = 0, sY = 0, sXX = 0, sYY = 0, sXY = 0; + + // Due to masking, it's important to account for fractional samples. + // For example, samples with a 50% mask are counted as a half sample. + double num_samples = 0; + + for (int r = 0; r < num_samples_y_; ++r) { + for (int c = 0; c < num_samples_x_; ++c) { + // Use the pre-computed image1 position. + Vec2 image1_position(pattern_positions_(r, c, 0), + pattern_positions_(r, c, 1)); + + double mask_value = 1.0; + if (options_.image1_mask != NULL) { + mask_value = pattern_mask_(r, c); + if (mask_value == 0.0) { + continue; + } + } + + // Compute the location of the destination pixel. + double image2_position[2]; + warp_.Forward(warp_parameters, + image1_position[0], + image1_position[1], + &image2_position[0], + &image2_position[1]); + + double x = pattern_and_gradient_(r, c); + double y = SampleLinear(image_and_gradient2_, + image2_position[1], // SampleLinear is r, c. + image2_position[0]); + + // Weight the signals by the mask, if one is present. + if (options_.image1_mask != NULL) { + x *= mask_value; + y *= mask_value; + num_samples += mask_value; + } else { + num_samples++; + } + sX += x; + sY += y; + sXX += x*x; + sYY += y*y; + sXY += x*y; + } + } + // Normalize. + sX /= num_samples; + sY /= num_samples; + sXX /= num_samples; + sYY /= num_samples; + sXY /= num_samples; + + double var_x = sXX - sX*sX; + double var_y = sYY - sY*sY; + double covariance_xy = sXY - sX*sY; + + double correlation = covariance_xy / sqrt(var_x * var_y); + LG << "Covariance xy: " << covariance_xy + << ", var 1: " << var_x << ", var 2: " << var_y + << ", correlation: " << correlation; + return correlation; + } + + private: + const TrackRegionOptions &options_; + const FloatImage &image_and_gradient1_; + const FloatImage &image_and_gradient2_; + const Mat3 &canonical_to_image1_; + int num_samples_x_; + int num_samples_y_; + const Warp &warp_; + double src_mean_; + FloatImage pattern_and_gradient_; + + // This contains the position from where the cached pattern samples were + // taken from. This is also used to warp from src to dest without going from + // canonical pixels to src first. + FloatImage pattern_positions_; + + FloatImage pattern_mask_; +}; + +template<typename Warp> +class WarpRegularizingCostFunctor { + public: + WarpRegularizingCostFunctor(const TrackRegionOptions &options, + const double *x1, + const double *y1, + const double *x2_original, + const double *y2_original, + const Warp &warp) + : options_(options), + x1_(x1), + y1_(y1), + x2_original_(x2_original), + y2_original_(y2_original), + warp_(warp) { + // Compute the centroid of the first guess quad. + // TODO(keir): Use Quad class here. + original_centroid_[0] = 0.0; + original_centroid_[1] = 0.0; + for (int i = 0; i < 4; ++i) { + original_centroid_[0] += x2_original[i]; + original_centroid_[1] += y2_original[i]; + } + original_centroid_[0] /= 4; + original_centroid_[1] /= 4; + } + + template<typename T> + bool operator()(const T *warp_parameters, T *residuals) const { + T dst_centroid[2] = { T(0.0), T(0.0) }; + for (int i = 0; i < 4; ++i) { + T image1_position[2] = { T(x1_[i]), T(y1_[i]) }; + T image2_position[2]; + warp_.Forward(warp_parameters, + T(x1_[i]), + T(y1_[i]), + &image2_position[0], + &image2_position[1]); + + // Subtract the positions. Note that this ignores the centroids. + residuals[2 * i + 0] = image2_position[0] - image1_position[0]; + residuals[2 * i + 1] = image2_position[1] - image1_position[1]; + + // Accumulate the dst centroid. + dst_centroid[0] += image2_position[0]; + dst_centroid[1] += image2_position[1]; + } + dst_centroid[0] /= T(4.0); + dst_centroid[1] /= T(4.0); + + // Adjust for the centroids. + for (int i = 0; i < 4; ++i) { + residuals[2 * i + 0] += T(original_centroid_[0]) - dst_centroid[0]; + residuals[2 * i + 1] += T(original_centroid_[1]) - dst_centroid[1]; + } + + // Reweight the residuals. + for (int i = 0; i < 8; ++i) { + residuals[i] *= T(options_.regularization_coefficient); + } + + return true; + } + + const TrackRegionOptions &options_; + const double *x1_; + const double *y1_; + const double *x2_original_; + const double *y2_original_; + double original_centroid_[2]; + const Warp &warp_; +}; + +// Compute the warp from rectangular coordinates, where one corner is the +// origin, and the opposite corner is at (num_samples_x, num_samples_y). +Mat3 ComputeCanonicalHomography(const double *x1, + const double *y1, + int num_samples_x, + int num_samples_y) { + Mat canonical(2, 4); + canonical << 0, num_samples_x, num_samples_x, 0, + 0, 0, num_samples_y, num_samples_y; + + Mat xy1(2, 4); + xy1 << x1[0], x1[1], x1[2], x1[3], + y1[0], y1[1], y1[2], y1[3]; + + Mat3 H; + if (!Homography2DFromCorrespondencesLinear(canonical, xy1, &H, 1e-12)) { + LG << "Couldn't construct homography."; + } + return H; +} + +class Quad { + public: + Quad(const double *x, const double *y) : x_(x), y_(y) { + // Compute the centroid and store it. + centroid_ = Vec2(0.0, 0.0); + for (int i = 0; i < 4; ++i) { + centroid_ += Vec2(x_[i], y_[i]); + } + centroid_ /= 4.0; + } + + // The centroid of the four points representing the quad. + const Vec2& Centroid() const { + return centroid_; + } + + // The average magnitude of the four points relative to the centroid. + double Scale() const { + double scale = 0.0; + for (int i = 0; i < 4; ++i) { + scale += (Vec2(x_[i], y_[i]) - Centroid()).norm(); + } + return scale / 4.0; + } + + Vec2 CornerRelativeToCentroid(int i) const { + return Vec2(x_[i], y_[i]) - centroid_; + } + + private: + const double *x_; + const double *y_; + Vec2 centroid_; +}; + +struct TranslationWarp { + TranslationWarp(const double *x1, const double *y1, + const double *x2, const double *y2) { + Vec2 t = Quad(x2, y2).Centroid() - Quad(x1, y1).Centroid(); + parameters[0] = t[0]; + parameters[1] = t[1]; + } + + template<typename T> + void Forward(const T *warp_parameters, + const T &x1, const T& y1, T *x2, T* y2) const { + *x2 = x1 + warp_parameters[0]; + *y2 = y1 + warp_parameters[1]; + } + + // Translation x, translation y. + enum { NUM_PARAMETERS = 2 }; + double parameters[NUM_PARAMETERS]; +}; + +struct TranslationScaleWarp { + TranslationScaleWarp(const double *x1, const double *y1, + const double *x2, const double *y2) + : q1(x1, y1) { + Quad q2(x2, y2); + + // The difference in centroids is the best guess for translation. + Vec2 t = q2.Centroid() - q1.Centroid(); + parameters[0] = t[0]; + parameters[1] = t[1]; + + // The difference in scales is the estimate for the scale. + parameters[2] = 1.0 - q2.Scale() / q1.Scale(); + } + + // The strange way of parameterizing the translation and scaling is to make + // the knobs that the optimizer sees easy to adjust. This is less important + // for the scaling case than the rotation case. + template<typename T> + void Forward(const T *warp_parameters, + const T &x1, const T& y1, T *x2, T* y2) const { + // Make the centroid of Q1 the origin. + const T x1_origin = x1 - q1.Centroid()(0); + const T y1_origin = y1 - q1.Centroid()(1); + + // Scale uniformly about the origin. + const T scale = 1.0 + warp_parameters[2]; + const T x1_origin_scaled = scale * x1_origin; + const T y1_origin_scaled = scale * y1_origin; + + // Translate back into the space of Q1 (but scaled). + const T x1_scaled = x1_origin_scaled + q1.Centroid()(0); + const T y1_scaled = y1_origin_scaled + q1.Centroid()(1); + + // Translate into the space of Q2. + *x2 = x1_scaled + warp_parameters[0]; + *y2 = y1_scaled + warp_parameters[1]; + } + + // Translation x, translation y, scale. + enum { NUM_PARAMETERS = 3 }; + double parameters[NUM_PARAMETERS]; + + Quad q1; +}; + +// Assumes the given points are already zero-centroid and the same size. +Mat2 OrthogonalProcrustes(const Mat2 &correlation_matrix) { + Eigen::JacobiSVD<Mat2> svd(correlation_matrix, + Eigen::ComputeFullU | Eigen::ComputeFullV); + return svd.matrixV() * svd.matrixU().transpose(); +} + +struct TranslationRotationWarp { + TranslationRotationWarp(const double *x1, const double *y1, + const double *x2, const double *y2) + : q1(x1, y1) { + Quad q2(x2, y2); + + // The difference in centroids is the best guess for translation. + Vec2 t = q2.Centroid() - q1.Centroid(); + parameters[0] = t[0]; + parameters[1] = t[1]; + + // Obtain the rotation via orthorgonal procrustes. + Mat2 correlation_matrix; + for (int i = 0; i < 4; ++i) { + correlation_matrix += q1.CornerRelativeToCentroid(i) * + q2.CornerRelativeToCentroid(i).transpose(); + } + Mat2 R = OrthogonalProcrustes(correlation_matrix); + parameters[2] = atan2(R(1, 0), R(0, 0)); + + LG << "Correlation_matrix:\n" << correlation_matrix; + LG << "R:\n" << R; + LG << "Theta:" << parameters[2]; + } + + // The strange way of parameterizing the translation and rotation is to make + // the knobs that the optimizer sees easy to adjust. The reason is that while + // it is always the case that it is possible to express composed rotations + // and translations as a single translation and rotation, the numerical + // values needed for the composition are often large in magnitude. This is + // enough to throw off any minimizer, since it must do the equivalent of + // compose rotations and translations. + // + // Instead, use the parameterization below that offers a parameterization + // that exposes the degrees of freedom in a way amenable to optimization. + template<typename T> + void Forward(const T *warp_parameters, + const T &x1, const T& y1, T *x2, T* y2) const { + // Make the centroid of Q1 the origin. + const T x1_origin = x1 - q1.Centroid()(0); + const T y1_origin = y1 - q1.Centroid()(1); + + // Rotate about the origin (i.e. centroid of Q1). + const T theta = warp_parameters[2]; + const T costheta = cos(theta); + const T sintheta = sin(theta); + const T x1_origin_rotated = costheta * x1_origin - sintheta * y1_origin; + const T y1_origin_rotated = sintheta * x1_origin + costheta * y1_origin; + + // Translate back into the space of Q1 (but scaled). + const T x1_rotated = x1_origin_rotated + q1.Centroid()(0); + const T y1_rotated = y1_origin_rotated + q1.Centroid()(1); + + // Translate into the space of Q2. + *x2 = x1_rotated + warp_parameters[0]; + *y2 = y1_rotated + warp_parameters[1]; + } + + // Translation x, translation y, rotation about the center of Q1 degrees. + enum { NUM_PARAMETERS = 3 }; + double parameters[NUM_PARAMETERS]; + + Quad q1; +}; + +struct TranslationRotationScaleWarp { + TranslationRotationScaleWarp(const double *x1, const double *y1, + const double *x2, const double *y2) + : q1(x1, y1) { + Quad q2(x2, y2); + + // The difference in centroids is the best guess for translation. + Vec2 t = q2.Centroid() - q1.Centroid(); + parameters[0] = t[0]; + parameters[1] = t[1]; + + // The difference in scales is the estimate for the scale. + parameters[2] = 1.0 - q2.Scale() / q1.Scale(); + + // Obtain the rotation via orthorgonal procrustes. + Mat2 correlation_matrix; + for (int i = 0; i < 4; ++i) { + correlation_matrix += q1.CornerRelativeToCentroid(i) * + q2.CornerRelativeToCentroid(i).transpose(); + } + Mat2 R = OrthogonalProcrustes(correlation_matrix); + parameters[3] = atan2(R(1, 0), R(0, 0)); + + LG << "Correlation_matrix:\n" << correlation_matrix; + LG << "R:\n" << R; + LG << "Theta:" << parameters[3]; + } + + // The strange way of parameterizing the translation and rotation is to make + // the knobs that the optimizer sees easy to adjust. The reason is that while + // it is always the case that it is possible to express composed rotations + // and translations as a single translation and rotation, the numerical + // values needed for the composition are often large in magnitude. This is + // enough to throw off any minimizer, since it must do the equivalent of + // compose rotations and translations. + // + // Instead, use the parameterization below that offers a parameterization + // that exposes the degrees of freedom in a way amenable to optimization. + template<typename T> + void Forward(const T *warp_parameters, + const T &x1, const T& y1, T *x2, T* y2) const { + // Make the centroid of Q1 the origin. + const T x1_origin = x1 - q1.Centroid()(0); + const T y1_origin = y1 - q1.Centroid()(1); + + // Rotate about the origin (i.e. centroid of Q1). + const T theta = warp_parameters[3]; + const T costheta = cos(theta); + const T sintheta = sin(theta); + const T x1_origin_rotated = costheta * x1_origin - sintheta * y1_origin; + const T y1_origin_rotated = sintheta * x1_origin + costheta * y1_origin; + + // Scale uniformly about the origin. + const T scale = 1.0 + warp_parameters[2]; + const T x1_origin_rotated_scaled = scale * x1_origin_rotated; + const T y1_origin_rotated_scaled = scale * y1_origin_rotated; + + // Translate back into the space of Q1 (but scaled and rotated). + const T x1_rotated_scaled = x1_origin_rotated_scaled + q1.Centroid()(0); + const T y1_rotated_scaled = y1_origin_rotated_scaled + q1.Centroid()(1); + + // Translate into the space of Q2. + *x2 = x1_rotated_scaled + warp_parameters[0]; + *y2 = y1_rotated_scaled + warp_parameters[1]; + } + + // Translation x, translation y, rotation about the center of Q1 degrees, + // scale. + enum { NUM_PARAMETERS = 4 }; + double parameters[NUM_PARAMETERS]; + + Quad q1; +}; + +struct AffineWarp { + AffineWarp(const double *x1, const double *y1, + const double *x2, const double *y2) + : q1(x1, y1) { + Quad q2(x2, y2); + + // The difference in centroids is the best guess for translation. + Vec2 t = q2.Centroid() - q1.Centroid(); + parameters[0] = t[0]; + parameters[1] = t[1]; + + // Estimate the four affine parameters with the usual least squares. + Mat Q1(8, 4); + Vec Q2(8); + for (int i = 0; i < 4; ++i) { + Vec2 v1 = q1.CornerRelativeToCentroid(i); + Vec2 v2 = q2.CornerRelativeToCentroid(i); + + Q1.row(2 * i + 0) << v1[0], v1[1], 0, 0 ; + Q1.row(2 * i + 1) << 0, 0, v1[0], v1[1]; + + Q2(2 * i + 0) = v2[0]; + Q2(2 * i + 1) = v2[1]; + } + + // TODO(keir): Check solution quality. + Vec4 a = Q1.jacobiSvd(Eigen::ComputeThinU | Eigen::ComputeThinV).solve(Q2); + parameters[2] = a[0]; + parameters[3] = a[1]; + parameters[4] = a[2]; + parameters[5] = a[3]; + + LG << "a:" << a.transpose(); + LG << "t:" << t.transpose(); + } + + // See comments in other parameterizations about why the centroid is used. + template<typename T> + void Forward(const T *p, const T &x1, const T& y1, T *x2, T* y2) const { + // Make the centroid of Q1 the origin. + const T x1_origin = x1 - q1.Centroid()(0); + const T y1_origin = y1 - q1.Centroid()(1); + + // Apply the affine transformation. + const T x1_origin_affine = p[2] * x1_origin + p[3] * y1_origin; + const T y1_origin_affine = p[4] * x1_origin + p[5] * y1_origin; + + // Translate back into the space of Q1 (but affine transformed). + const T x1_affine = x1_origin_affine + q1.Centroid()(0); + const T y1_affine = y1_origin_affine + q1.Centroid()(1); + + // Translate into the space of Q2. + *x2 = x1_affine + p[0]; + *y2 = y1_affine + p[1]; + } + + // Translation x, translation y, rotation about the center of Q1 degrees, + // scale. + enum { NUM_PARAMETERS = 6 }; + double parameters[NUM_PARAMETERS]; + + Quad q1; +}; + +struct HomographyWarp { + HomographyWarp(const double *x1, const double *y1, + const double *x2, const double *y2) { + Mat quad1(2, 4); + quad1 << x1[0], x1[1], x1[2], x1[3], + y1[0], y1[1], y1[2], y1[3]; + + Mat quad2(2, 4); + quad2 << x2[0], x2[1], x2[2], x2[3], + y2[0], y2[1], y2[2], y2[3]; + + Mat3 H; + if (!Homography2DFromCorrespondencesLinear(quad1, quad2, &H, 1e-12)) { + LG << "Couldn't construct homography."; + } + + // Assume H(2, 2) != 0, and fix scale at H(2, 2) == 1.0. + H /= H(2, 2); + + // Assume H is close to identity, so subtract out the diagonal. + H(0, 0) -= 1.0; + H(1, 1) -= 1.0; + + CHECK_NE(H(2, 2), 0.0) << H; + for (int i = 0; i < 8; ++i) { + parameters[i] = H(i / 3, i % 3); + LG << "Parameters[" << i << "]: " << parameters[i]; + } + } + + template<typename T> + static void Forward(const T *p, + const T &x1, const T& y1, T *x2, T* y2) { + // Homography warp with manual 3x3 matrix multiply. + const T xx2 = (1.0 + p[0]) * x1 + p[1] * y1 + p[2]; + const T yy2 = p[3] * x1 + (1.0 + p[4]) * y1 + p[5]; + const T zz2 = p[6] * x1 + p[7] * y1 + 1.0; + *x2 = xx2 / zz2; + *y2 = yy2 / zz2; + } + + enum { NUM_PARAMETERS = 8 }; + double parameters[NUM_PARAMETERS]; +}; + +// Determine the number of samples to use for x and y. Quad winding goes: +// +// 0 1 +// 3 2 +// +// The idea is to take the maximum x or y distance. This may be oversampling. +// TODO(keir): Investigate the various choices; perhaps average is better? +void PickSampling(const double *x1, const double *y1, + const double *x2, const double *y2, + int *num_samples_x, int *num_samples_y) { + Vec2 a0(x1[0], y1[0]); + Vec2 a1(x1[1], y1[1]); + Vec2 a2(x1[2], y1[2]); + Vec2 a3(x1[3], y1[3]); + + Vec2 b0(x1[0], y1[0]); + Vec2 b1(x1[1], y1[1]); + Vec2 b2(x1[2], y1[2]); + Vec2 b3(x1[3], y1[3]); + + double x_dimensions[4] = { + (a1 - a0).norm(), + (a3 - a2).norm(), + (b1 - b0).norm(), + (b3 - b2).norm() + }; + + double y_dimensions[4] = { + (a3 - a0).norm(), + (a1 - a2).norm(), + (b3 - b0).norm(), + (b1 - b2).norm() + }; + const double kScaleFactor = 1.0; + *num_samples_x = static_cast<int>( + kScaleFactor * *std::max_element(x_dimensions, x_dimensions + 4)); + *num_samples_y = static_cast<int>( + kScaleFactor * *std::max_element(y_dimensions, y_dimensions + 4)); + LG << "Automatic num_samples_x: " << *num_samples_x + << ", num_samples_y: " << *num_samples_y; +} + +bool SearchAreaTooBigForDescent(const FloatImage &image2, + const double *x2, const double *y2) { + // TODO(keir): Check the bounds and enable only when it makes sense. + return true; +} + +bool PointOnRightHalfPlane(const Vec2 &a, const Vec2 &b, double x, double y) { + Vec2 ba = b - a; + return ((Vec2(x, y) - b).transpose() * Vec2(-ba.y(), ba.x())) > 0; +} + +// Determine if a point is in a quad. The quad is arranged as: +// +// +-------> x +// | +// | a0------a1 +// | | | +// | | | +// | | | +// | a3------a2 +// v +// y +// +// The implementation does up to four half-plane comparisons. +bool PointInQuad(const double *xs, const double *ys, double x, double y) { + Vec2 a0(xs[0], ys[0]); + Vec2 a1(xs[1], ys[1]); + Vec2 a2(xs[2], ys[2]); + Vec2 a3(xs[3], ys[3]); + + return PointOnRightHalfPlane(a0, a1, x, y) && + PointOnRightHalfPlane(a1, a2, x, y) && + PointOnRightHalfPlane(a2, a3, x, y) && + PointOnRightHalfPlane(a3, a0, x, y); +} + +// This makes it possible to map between Eigen float arrays and FloatImage +// without using comparisons. +typedef Eigen::Array<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> FloatArray; + +// This creates a pattern in the frame of image2, from the pixel is image1, +// based on the initial guess represented by the two quads x1, y1, and x2, y2. +template<typename Warp> +void CreateBrutePattern(const double *x1, const double *y1, + const double *x2, const double *y2, + const FloatImage &image1, + const FloatImage *image1_mask, + FloatArray *pattern, + FloatArray *mask, + int *origin_x, + int *origin_y) { + // Get integer bounding box of quad2 in image2. + int min_x = static_cast<int>(floor(*std::min_element(x2, x2 + 4))); + int min_y = static_cast<int>(floor(*std::min_element(y2, y2 + 4))); + int max_x = static_cast<int>(ceil (*std::max_element(x2, x2 + 4))); + int max_y = static_cast<int>(ceil (*std::max_element(y2, y2 + 4))); + + int w = max_x - min_x; + int h = max_y - min_y; + + pattern->resize(h, w); + mask->resize(h, w); + + Warp inverse_warp(x2, y2, x1, y1); + + // r,c are in the coordinate frame of image2. + for (int r = min_y; r < max_y; ++r) { + for (int c = min_x; c < max_x; ++c) { + // i and j are in the coordinate frame of the pattern in image2. + int i = r - min_y; + int j = c - min_x; + + double dst_x = c; + double dst_y = r; + double src_x; + double src_y; + inverse_warp.Forward(inverse_warp.parameters, + dst_x, dst_y, + &src_x, &src_y); + + if (PointInQuad(x1, y1, src_x, src_y)) { + (*pattern)(i, j) = SampleLinear(image1, src_y, src_x); + (*mask)(i, j) = 1.0; + if (image1_mask) { + (*mask)(i, j) = SampleLinear(*image1_mask, src_y, src_x);; + } + } else { + (*pattern)(i, j) = 0.0; + (*mask)(i, j) = 0.0; + } + } + } + *origin_x = min_x; + *origin_y = min_y; +} + +// Compute a translation-only estimate of the warp, using brute force search. A +// smarter implementation would use the FFT to compute the normalized cross +// correlation. Instead, this is a dumb implementation. Surprisingly, it is +// fast enough in practice. +// +// TODO(keir): The normalization is less effective for the brute force search +// than it is with the Ceres solver. It's unclear if this is a bug or due to +// the original frame being too different from the reprojected reference in the +// destination frame. +// +// The likely solution is to use the previous frame, instead of the original +// pattern, when doing brute initialization. Unfortunately that implies a +// totally different warping interface, since access to more than a the source +// and current destination frame is necessary. +template<typename Warp> +void BruteTranslationOnlyInitialize(const FloatImage &image1, + const FloatImage *image1_mask, + const FloatImage &image2, + const int num_extra_points, + const bool use_normalized_intensities, + const double *x1, const double *y1, + double *x2, double *y2) { + // Create the pattern to match in the space of image2, assuming our inital + // guess isn't too far from the template in image1. If there is no image1 + // mask, then the resulting mask is binary. + FloatArray pattern; + FloatArray mask; + int origin_x = -1, origin_y = -1; + CreateBrutePattern<Warp>(x1, y1, x2, y2, + image1, image1_mask, + &pattern, &mask, + &origin_x, &origin_y); + + // For normalization, premultiply the pattern by the inverse pattern mean. + double mask_sum = 1.0; + if (use_normalized_intensities) { + mask_sum = mask.sum(); + double inverse_pattern_mean = mask_sum / ((mask * pattern).sum()); + pattern *= inverse_pattern_mean; + } + + // Use Eigen on the images via maps for strong vectorization. + Map<const FloatArray> search(image2.Data(), image2.Height(), image2.Width()); + + // Try all possible locations inside the search area. Yes, everywhere. + // + // TODO(keir): There are a number of possible optimizations here. One choice + // is to make a grid and only try one out of every N possible samples. + // + // Another, slightly more clever idea, is to compute some sort of spatial + // frequency distribution of the pattern patch. If the spatial resolution is + // high (e.g. a grating pattern or fine lines) then checking every possible + // translation is necessary, since a 1-pixel shift may induce a massive + // change in the cost function. If the image is a blob or splotch with blurry + // edges, then fewer samples are necessary since a few pixels offset won't + // change the cost function much. + double best_sad = std::numeric_limits<double>::max(); + int best_r = -1; + int best_c = -1; + int w = pattern.cols(); + int h = pattern.rows(); + for (int r = 0; r < (image2.Height() - h); ++r) { + for (int c = 0; c < (image2.Width() - w); ++c) { + // Compute the weighted sum of absolute differences, Eigen style. Note + // that the block from the search image is never stored in a variable, to + // avoid copying overhead and permit inlining. + double sad; + if (use_normalized_intensities) { + // TODO(keir): It's really dumb to recompute the search mean for every + // shift. A smarter implementation would use summed area tables + // instead, reducing the mean calculation to an O(1) operation. + double inverse_search_mean = + mask_sum / ((mask * search.block(r, c, h, w)).sum()); + sad = (mask * (pattern - (search.block(r, c, h, w) * + inverse_search_mean))).abs().sum(); + } else { + sad = (mask * (pattern - search.block(r, c, h, w))).abs().sum(); + } + if (sad < best_sad) { + best_r = r; + best_c = c; + best_sad = sad; + } + } + } + CHECK_NE(best_r, -1); + CHECK_NE(best_c, -1); + + LG << "Brute force translation found a shift. " + << "best_c: " << best_c << ", best_r: " << best_r << ", " + << "origin_x: " << origin_x << ", origin_y: " << origin_y << ", " + << "dc: " << (best_c - origin_x) << ", " + << "dr: " << (best_r - origin_y) + << ", tried " << ((image2.Height() - h) * (image2.Width() - w)) + << " shifts."; + + // Apply the shift. + for (int i = 0; i < 4 + num_extra_points; ++i) { + x2[i] += best_c - origin_x; + y2[i] += best_r - origin_y; + } +} + +} // namespace + +template<typename Warp> +void TemplatedTrackRegion(const FloatImage &image1, + const FloatImage &image2, + const double *x1, const double *y1, + const TrackRegionOptions &options, + double *x2, double *y2, + TrackRegionResult *result) { + for (int i = 0; i < 4; ++i) { + LG << "P" << i << ": (" << x1[i] << ", " << y1[i] << "); guess (" + << x2[i] << ", " << y2[i] << "); (dx, dy): (" << (x2[i] - x1[i]) << ", " + << (y2[i] - y1[i]) << ")."; + } + if (options.use_normalized_intensities) { + LG << "Using normalized intensities."; + } + + // Bail early if the points are already outside. + if (!AllInBounds(image1, x1, y1)) { + result->termination = TrackRegionResult::SOURCE_OUT_OF_BOUNDS; + return; + } + if (!AllInBounds(image2, x2, y2)) { + result->termination = TrackRegionResult::DESTINATION_OUT_OF_BOUNDS; + return; + } + // TODO(keir): Check quads to ensure there is some area. + + // Keep a copy of the "original" guess for regularization. + double x2_original[4]; + double y2_original[4]; + for (int i = 0; i < 4; ++i) { + x2_original[i] = x2[i]; + y2_original[i] = y2[i]; + } + + // Prepare the image and gradient. + Array3Df image_and_gradient1; + Array3Df image_and_gradient2; + BlurredImageAndDerivativesChannels(image1, options.sigma, + &image_and_gradient1); + BlurredImageAndDerivativesChannels(image2, options.sigma, + &image_and_gradient2); + + // Possibly do a brute-force translation-only initialization. + if (SearchAreaTooBigForDescent(image2, x2, y2) && + options.use_brute_initialization) { + LG << "Running brute initialization..."; + BruteTranslationOnlyInitialize<Warp>(image_and_gradient1, + options.image1_mask, + image2, + options.num_extra_points, + options.use_normalized_intensities, + x1, y1, x2, y2); + for (int i = 0; i < 4; ++i) { + LG << "P" << i << ": (" << x1[i] << ", " << y1[i] << "); brute (" + << x2[i] << ", " << y2[i] << "); (dx, dy): (" << (x2[i] - x1[i]) + << ", " << (y2[i] - y1[i]) << ")."; + } + } + + // Prepare the initial warp parameters from the four correspondences. + // Note: This must happen after the brute initialization runs, since the + // brute initialization mutates x2 and y2 in place. + Warp warp(x1, y1, x2, y2); + + // Decide how many samples to use in the x and y dimensions. + int num_samples_x; + int num_samples_y; + PickSampling(x1, y1, x2, y2, &num_samples_x, &num_samples_y); + + + // Compute the warp from rectangular coordinates. + Mat3 canonical_homography = ComputeCanonicalHomography(x1, y1, + num_samples_x, + num_samples_y); + + ceres::Problem problem; + + // Construct the warp cost function. AutoDiffCostFunction takes ownership. + PixelDifferenceCostFunctor<Warp> *pixel_difference_cost_function = + new PixelDifferenceCostFunctor<Warp>(options, + image_and_gradient1, + image_and_gradient2, + canonical_homography, + num_samples_x, + num_samples_y, + warp); + problem.AddResidualBlock( + new ceres::AutoDiffCostFunction< + PixelDifferenceCostFunctor<Warp>, + ceres::DYNAMIC, + Warp::NUM_PARAMETERS>(pixel_difference_cost_function, + num_samples_x * num_samples_y), + NULL, + warp.parameters); + + // Construct the regularizing cost function + if (options.regularization_coefficient != 0.0) { + WarpRegularizingCostFunctor<Warp> *regularizing_warp_cost_function = + new WarpRegularizingCostFunctor<Warp>(options, + x1, y2, + x2_original, + y2_original, + warp); + + problem.AddResidualBlock( + new ceres::AutoDiffCostFunction< + WarpRegularizingCostFunctor<Warp>, + 8 /* num_residuals */, + Warp::NUM_PARAMETERS>(regularizing_warp_cost_function), + NULL, + warp.parameters); + } + + // Configure the solve. + ceres::Solver::Options solver_options; + solver_options.linear_solver_type = ceres::DENSE_QR; + solver_options.max_num_iterations = options.max_iterations; + solver_options.update_state_every_iteration = true; + solver_options.parameter_tolerance = 1e-16; + solver_options.function_tolerance = 1e-16; + + // Prevent the corners from going outside the destination image. + BoundaryCheckingCallback<Warp> callback(image2, warp, x1, y1); + solver_options.callbacks.push_back(&callback); + + // Run the solve. + ceres::Solver::Summary summary; + ceres::Solve(solver_options, &problem, &summary); + + LG << "Summary:\n" << summary.FullReport(); + + // Update the four points with the found solution; if the solver failed, then + // the warp parameters are the identity (so ignore failure). + // + // Also warp any extra points on the end of the array. + for (int i = 0; i < 4 + options.num_extra_points; ++i) { + warp.Forward(warp.parameters, x1[i], y1[i], x2 + i, y2 + i); + LG << "Warped point " << i << ": (" << x1[i] << ", " << y1[i] << ") -> (" + << x2[i] << ", " << y2[i] << "); (dx, dy): (" << (x2[i] - x1[i]) << ", " + << (y2[i] - y1[i]) << ")."; + } + + // TODO(keir): Update the result statistics. + // TODO(keir): Add a normalize-cross-correlation variant. + + CHECK_NE(summary.termination_type, ceres::USER_ABORT) << "Libmv bug."; + if (summary.termination_type == ceres::USER_ABORT) { + result->termination = TrackRegionResult::FELL_OUT_OF_BOUNDS; + return; + } +#define HANDLE_TERMINATION(termination_enum) \ + if (summary.termination_type == ceres::termination_enum) { \ + result->termination = TrackRegionResult::termination_enum; \ + return; \ + } + + // Avoid computing correlation for tracking failures. + HANDLE_TERMINATION(DID_NOT_RUN); + HANDLE_TERMINATION(NUMERICAL_FAILURE); + + // Otherwise, run a final correlation check. + if (options.minimum_correlation > 0.0) { + result->correlation = pixel_difference_cost_function-> + PearsonProductMomentCorrelationCoefficient(warp.parameters); + if (result->correlation < options.minimum_correlation) { + LG << "Failing with insufficient correlation."; + result->termination = TrackRegionResult::INSUFFICIENT_CORRELATION; + return; + } + } + + HANDLE_TERMINATION(PARAMETER_TOLERANCE); + HANDLE_TERMINATION(FUNCTION_TOLERANCE); + HANDLE_TERMINATION(GRADIENT_TOLERANCE); + HANDLE_TERMINATION(NO_CONVERGENCE); +#undef HANDLE_TERMINATION +}; + +void TrackRegion(const FloatImage &image1, + const FloatImage &image2, + const double *x1, const double *y1, + const TrackRegionOptions &options, + double *x2, double *y2, + TrackRegionResult *result) { + // Enum is necessary due to templated nature of autodiff. +#define HANDLE_MODE(mode_enum, mode_type) \ + if (options.mode == TrackRegionOptions::mode_enum) { \ + TemplatedTrackRegion<mode_type>(image1, image2, \ + x1, y1, \ + options, \ + x2, y2, \ + result); \ + return; \ + } + HANDLE_MODE(TRANSLATION, TranslationWarp); + HANDLE_MODE(TRANSLATION_SCALE, TranslationScaleWarp); + HANDLE_MODE(TRANSLATION_ROTATION, TranslationRotationWarp); + HANDLE_MODE(TRANSLATION_ROTATION_SCALE, TranslationRotationScaleWarp); + HANDLE_MODE(AFFINE, AffineWarp); + HANDLE_MODE(HOMOGRAPHY, HomographyWarp); +#undef HANDLE_MODE +} + +bool SamplePlanarPatch(const FloatImage &image, + const double *xs, const double *ys, + int num_samples_x, int num_samples_y, + FloatImage *patch, + double *warped_position_x, double *warped_position_y) { + // Bail early if the points are outside the image. + if (!AllInBounds(image, xs, ys)) { + LG << "Can't sample patch: out of bounds."; + return false; + } + + // Make the patch have the appropriate size, and match the depth of image. + patch->Resize(num_samples_y, num_samples_x, image.Depth()); + + // Compute the warp from rectangular coordinates. + Mat3 canonical_homography = ComputeCanonicalHomography(xs, ys, + num_samples_x, + num_samples_y); + + // Walk over the coordinates in the canonical space, sampling from the image + // in the original space and copying the result into the patch. + for (int r = 0; r < num_samples_y; ++r) { + for (int c = 0; c < num_samples_x; ++c) { + Vec3 image_position = canonical_homography * Vec3(c, r, 1); + image_position /= image_position(2); + SampleLinear(image, image_position(1), + image_position(0), + &(*patch)(r, c, 0)); + } + } + + Vec3 warped_position = canonical_homography.inverse() * Vec3(xs[4], ys[4], 1); + warped_position /= warped_position(2); + + *warped_position_x = warped_position(0); + *warped_position_y = warped_position(1); + + return true; +} + +} // namespace libmv diff --git a/extern/libmv/libmv/tracking/track_region.h b/extern/libmv/libmv/tracking/track_region.h new file mode 100644 index 00000000000..0de11239da6 --- /dev/null +++ b/extern/libmv/libmv/tracking/track_region.h @@ -0,0 +1,146 @@ +// Copyright (c) 2012 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. + +#ifndef LIBMV_TRACKING_TRACK_REGION_H_ + +// Necessary for M_E when building with MSVC. +#define _USE_MATH_DEFINES + +#include "libmv/tracking/esm_region_tracker.h" + +#include "libmv/image/image.h" +#include "libmv/image/sample.h" +#include "libmv/numeric/numeric.h" + +namespace libmv { + +struct TrackRegionOptions { + TrackRegionOptions(); + + enum Mode { + TRANSLATION, + TRANSLATION_ROTATION, + TRANSLATION_SCALE, + TRANSLATION_ROTATION_SCALE, + AFFINE, + HOMOGRAPHY, + }; + Mode mode; + + double minimum_correlation; + int max_iterations; + + // Use the "Efficient Second-order Minimization" scheme. This increases + // convergence speed at the cost of more per-iteration work. + bool use_esm; + + // If true, apply a brute-force translation-only search before attempting the + // full search. This is not enabled if the destination image ("image2") is + // too small; in that case eithen the basin of attraction is close enough + // that the nearby minima is correct, or the search area is too small. + bool use_brute_initialization; + + // If true, normalize the image patches by their mean before doing the sum of + // squared error calculation. This is reasonable since the effect of + // increasing light intensity is multiplicative on the pixel intensities. + // + // Note: This does nearly double the solving time, so it is not advised to + // turn this on all the time. + bool use_normalized_intensities; + + // The size in pixels of the blur kernel used to both smooth the image and + // take the image derivative. + double sigma; + + // Extra points that should get transformed by the warp. This is useful + // because the actual warp parameters are not exposed. + int num_extra_points; + + // For motion models other than translation, the optimizer sometimes has + // trouble deciding what to do around flat areas in the cost function. This + // leads to the optimizer picking poor solutions near the minimum. Visually, + // the effect is that the quad corners jiggle around, even though the center + // of the patch is well estimated. regularization_coefficient controls a term + // in the sum of squared error cost that makes it expensive for the optimizer + // to pick a warp that changes the shape of the patch dramatically (e.g. + // rotating, scaling, skewing, etc). + // + // In particular it adds an 8-residual cost function to the optimization, + // where each corner induces 2 residuals: the difference between the warped + // and the initial guess. However, the patch centroids are subtracted so that + // only patch distortions are penalized. + // + // If zero, no regularization is used. + double regularization_coefficient; + + // If non-null, this is used as the pattern mask. It should match the size of + // image1, even though only values inside the image1 quad are examined. The + // values must be in the range 0.0 to 0.1. + FloatImage *image1_mask; +}; + +struct TrackRegionResult { + enum Termination { + // Ceres termination types, duplicated; though, not the int values. + PARAMETER_TOLERANCE, + FUNCTION_TOLERANCE, + GRADIENT_TOLERANCE, + NO_CONVERGENCE, + DID_NOT_RUN, + NUMERICAL_FAILURE, + + // Libmv specific errors. + SOURCE_OUT_OF_BOUNDS, + DESTINATION_OUT_OF_BOUNDS, + FELL_OUT_OF_BOUNDS, + INSUFFICIENT_CORRELATION, + CONFIGURATION_ERROR, + }; + Termination termination; + + int num_iterations; + double correlation; + + // Final parameters? + bool used_brute_translation_initialization; +}; + +// Always needs 4 correspondences. +void TrackRegion(const FloatImage &image1, + const FloatImage &image2, + const double *x1, const double *y1, + const TrackRegionOptions &options, + double *x2, double *y2, + TrackRegionResult *result); + +// Sample a "canonical" version of the passed planar patch, using bilinear +// sampling. The passed corners must be within the image, and have at least two +// pixels of border around them. (so e.g. a corner of the patch cannot lie +// directly on the edge of the image). Four corners are always required. All +// channels are interpolated. +bool SamplePlanarPatch(const FloatImage &image, + const double *xs, const double *ys, + int num_samples_x, int num_samples_y, + FloatImage *patch, + double *warped_position_x, double *warped_position_y); + +} // namespace libmv + +#endif // LIBMV_TRACKING_TRACK_REGION_H_ diff --git a/release/scripts/startup/bl_ui/space_clip.py b/release/scripts/startup/bl_ui/space_clip.py index d0f0f3de276..1cef4624a04 100644 --- a/release/scripts/startup/bl_ui/space_clip.py +++ b/release/scripts/startup/bl_ui/space_clip.py @@ -230,10 +230,9 @@ class CLIP_PT_tools_marker(CLIP_PT_tracking_panel, Panel): sub.prop(settings, "default_search_size") col.label(text="Tracker:") - col.prop(settings, "default_tracker", text="") - - if settings.default_tracker == 'KLT': - col.prop(settings, "default_pyramid_levels") + col.prop(settings, "default_motion_model") + col.prop(settings, "default_use_brute") + col.prop(settings, "default_use_normalization") col.prop(settings, "default_correlation_min") col.separator() @@ -575,10 +574,9 @@ class CLIP_PT_track_settings(CLIP_PT_tracking_panel, Panel): active = clip.tracking.tracks.active if active: - col.prop(active, "tracker") - - if active.tracker == 'KLT': - col.prop(active, "pyramid_levels") + col.prop(active, "motion_model") + col.prop(active, "use_brute") + col.prop(active, "use_normalization") col.prop(active, "correlation_min") col.separator() diff --git a/source/blender/blenkernel/BKE_tracking.h b/source/blender/blenkernel/BKE_tracking.h index 4d9a5746c87..8c28dd93a5e 100644 --- a/source/blender/blenkernel/BKE_tracking.h +++ b/source/blender/blenkernel/BKE_tracking.h @@ -48,7 +48,7 @@ struct Object; struct Scene; void BKE_tracking_init_settings(struct MovieTracking *tracking); -void BKE_tracking_clamp_track(struct MovieTrackingTrack *track, int event); +void BKE_tracking_clamp_marker(struct MovieTrackingMarker *marker, int event); void BKE_tracking_track_flag(struct MovieTrackingTrack *track, int area, int flag, int clear); struct MovieTrackingTrack *BKE_tracking_add_track(struct MovieTracking *tracking, struct ListBase *tracksbase, @@ -57,6 +57,7 @@ struct MovieTrackingMarker *BKE_tracking_insert_marker(struct MovieTrackingTrack struct MovieTrackingMarker *marker); void BKE_tracking_delete_marker(struct MovieTrackingTrack *track, int framenr); +void BKE_tracking_marker_pattern_minmax(struct MovieTrackingMarker *marker, float min[2], float max[2]); struct MovieTrackingMarker *BKE_tracking_get_marker(struct MovieTrackingTrack *track, int framenr); struct MovieTrackingMarker *BKE_tracking_ensure_marker(struct MovieTrackingTrack *track, int framenr); struct MovieTrackingMarker *BKE_tracking_exact_marker(struct MovieTrackingTrack *track, int framenr); @@ -70,12 +71,15 @@ void BKE_tracking_clear_path(struct MovieTrackingTrack *track, int ref_frame, in void BKE_tracking_join_tracks(struct MovieTrackingTrack *dst_track, struct MovieTrackingTrack *src_track); void BKE_tracking_free(struct MovieTracking *tracking); +struct ImBuf *BKE_tracking_sample_pattern_imbuf(int frame_width, int frame_height, + struct ImBuf *struct_ibuf, struct MovieTrackingMarker *marker, + int num_samples_x, int num_samples_y, float pos[2]); struct ImBuf *BKE_tracking_get_pattern_imbuf(struct ImBuf *ibuf, struct MovieTrackingTrack *track, - struct MovieTrackingMarker *marker, int margin, int anchored, - float pos[2], int origin[2]); + struct MovieTrackingMarker *marker, int anchored, int disable_channels); struct ImBuf *BKE_tracking_get_search_imbuf(struct ImBuf *ibuf, struct MovieTrackingTrack *track, - struct MovieTrackingMarker *marker, int margin, int anchored, - float pos[2], int origin[2]); + struct MovieTrackingMarker *marker, int anchored, int disable_channels); +struct ImBuf *BKE_tracking_track_mask_get(struct MovieTracking *tracking, struct MovieTrackingTrack *track, + struct MovieTrackingMarker *marker, int width, int height); void BKE_track_unique_name(struct ListBase *tracksbase, struct MovieTrackingTrack *track); @@ -189,7 +193,6 @@ void BKE_tracking_dopesheet_update(struct MovieTracking *tracking, int sort_meth #define CLAMP_PAT_POS 2 #define CLAMP_SEARCH_DIM 3 #define CLAMP_SEARCH_POS 4 -#define CLAMP_PYRAMID_LEVELS 5 #define TRACK_AREA_NONE -1 #define TRACK_AREA_POINT 1 diff --git a/source/blender/blenkernel/intern/movieclip.c b/source/blender/blenkernel/intern/movieclip.c index abdc8835d43..8b91ee29c59 100644 --- a/source/blender/blenkernel/intern/movieclip.c +++ b/source/blender/blenkernel/intern/movieclip.c @@ -1032,6 +1032,11 @@ void BKE_movieclip_update_scopes(MovieClip *clip, MovieClipUser *user, MovieClip scopes->track_preview = NULL; } + if (scopes->track_search) { + IMB_freeImBuf(scopes->track_search); + scopes->track_search = NULL; + } + scopes->marker = NULL; scopes->track = NULL; @@ -1052,7 +1057,7 @@ void BKE_movieclip_update_scopes(MovieClip *clip, MovieClipUser *user, MovieClip scopes->track_disabled = FALSE; if (ibuf && (ibuf->rect || ibuf->rect_float)) { - ImBuf *tmpibuf; + ImBuf *search_ibuf; MovieTrackingMarker undist_marker = *marker; if (user->render_flag & MCLIP_PROXY_RENDER_UNDISTORT) { @@ -1070,27 +1075,36 @@ void BKE_movieclip_update_scopes(MovieClip *clip, MovieClipUser *user, MovieClip undist_marker.pos[1] /= height * aspy; } - /* NOTE: margin should be kept in sync with value from ui_draw_but_TRACKPREVIEW */ - tmpibuf = BKE_tracking_get_pattern_imbuf(ibuf, track, &undist_marker, 3 /* margin */, - 1 /* anchor */, scopes->track_pos, NULL); + search_ibuf = BKE_tracking_get_search_imbuf(ibuf, track, &undist_marker, TRUE, TRUE); + + if (!search_ibuf->rect_float) { + /* sampling happens in float buffer */ + IMB_float_from_rect(search_ibuf); + } - if (tmpibuf->rect_float) - IMB_rect_from_float(tmpibuf); + scopes->undist_marker = undist_marker; + scopes->track_search = search_ibuf; - if (tmpibuf->rect) - scopes->track_preview = tmpibuf; - else - IMB_freeImBuf(tmpibuf); + scopes->frame_width = ibuf->x; + scopes->frame_height = ibuf->y; } IMB_freeImBuf(ibuf); } if ((track->flag & TRACK_LOCKED) == 0) { + float pat_min[2], pat_max[2]; + scopes->marker = marker; scopes->track = track; - scopes->slide_scale[0] = track->pat_max[0] - track->pat_min[0]; - scopes->slide_scale[1] = track->pat_max[1] - track->pat_min[1]; + + /* XXX: would work fine with non-transformed patterns, but would likely fail + * with transformed patterns, but that would be easier to debug when + * we'll have real pattern sampling (at least to test) */ + BKE_tracking_marker_pattern_minmax(marker, pat_min, pat_max); + + scopes->slide_scale[0] = pat_max[0] - pat_min[0]; + scopes->slide_scale[1] = pat_max[1] - pat_min[1]; } } } diff --git a/source/blender/blenkernel/intern/tracking.c b/source/blender/blenkernel/intern/tracking.c index fb78e89cc30..938bff81470 100644 --- a/source/blender/blenkernel/intern/tracking.c +++ b/source/blender/blenkernel/intern/tracking.c @@ -20,6 +20,7 @@ * * Contributor(s): Blender Foundation, * Sergey Sharybin + * Keir Mierle * * ***** END GPL LICENSE BLOCK ***** */ @@ -59,6 +60,8 @@ #include "IMB_imbuf_types.h" #include "IMB_imbuf.h" +#include "raskter.h" + #ifdef WITH_LIBMV # include "libmv-capi.h" #else @@ -73,6 +76,141 @@ static struct { ListBase tracks; } tracking_clipboard; +/*********************** space transformation functions *************************/ + +/* Three coordinate frames: Frame, Search, and Marker + * Two units: Pixels, Unified + * Notation: {coordinate frame}_{unit}; for example, "search_pixel" are search + * window relative coordinates in pixels, and "frame_unified" are unified 0..1 + * coordinates relative to the entire frame. + */ +static void unified_to_pixel(int frame_width, int frame_height, + const float unified_coords[2], float pixel_coords[2]) +{ + pixel_coords[0] = unified_coords[0] * frame_width; + pixel_coords[1] = unified_coords[1] * frame_height; +} + +static void marker_to_frame_unified(const MovieTrackingMarker *marker, const float marker_unified_coords[2], + float frame_unified_coords[2]) +{ + frame_unified_coords[0] = marker_unified_coords[0] + marker->pos[0]; + frame_unified_coords[1] = marker_unified_coords[1] + marker->pos[1]; +} + +static void marker_unified_to_frame_pixel_coordinates(int frame_width, int frame_height, + const MovieTrackingMarker *marker, + const float marker_unified_coords[2], float frame_pixel_coords[2]) +{ + marker_to_frame_unified(marker, marker_unified_coords, frame_pixel_coords); + unified_to_pixel(frame_width, frame_height, frame_pixel_coords, frame_pixel_coords); +} + +static void get_search_origin_frame_pixel(int frame_width, int frame_height, + const MovieTrackingMarker *marker, float frame_pixel[2]) +{ + /* Get the lower left coordinate of the search window and snap to pixel coordinates */ + marker_unified_to_frame_pixel_coordinates(frame_width, frame_height, marker, marker->search_min, frame_pixel); + frame_pixel[0] = (int)frame_pixel[0]; + frame_pixel[1] = (int)frame_pixel[1]; +} + +#ifdef WITH_LIBMV +static void pixel_to_unified(int frame_width, int frame_height, const float pixel_coords[2], float unified_coords[2]) +{ + unified_coords[0] = pixel_coords[0] / frame_width; + unified_coords[1] = pixel_coords[1] / frame_height; +} + +static void marker_unified_to_search_pixel(int frame_width, int frame_height, + const MovieTrackingMarker *marker, + const float marker_unified[2], float search_pixel[2]) +{ + float frame_pixel[2]; + float search_origin_frame_pixel[2]; + + marker_unified_to_frame_pixel_coordinates(frame_width, frame_height, marker, marker_unified, frame_pixel); + get_search_origin_frame_pixel(frame_width, frame_height, marker, search_origin_frame_pixel); + sub_v2_v2v2(search_pixel, frame_pixel, search_origin_frame_pixel); +} + +static void search_pixel_to_marker_unified(int frame_width, int frame_height, + const MovieTrackingMarker *marker, + const float search_pixel[2], float marker_unified[2]) +{ + float frame_unified[2]; + float search_origin_frame_pixel[2]; + + get_search_origin_frame_pixel(frame_width, frame_height, marker, search_origin_frame_pixel); + add_v2_v2v2(frame_unified, search_pixel, search_origin_frame_pixel); + pixel_to_unified(frame_width, frame_height, frame_unified, frame_unified); + + /* marker pos is in frame unified */ + sub_v2_v2v2(marker_unified, frame_unified, marker->pos); +} + +/* Each marker has 5 coordinates associated with it that get warped with + * tracking: the four corners ("pattern_corners"), and the cernter ("pos"). + * This function puts those 5 points into the appropriate frame for tracking + * (the "search" coordinate frame). + */ +static void get_marker_coords_for_tracking(int frame_width, int frame_height, + const MovieTrackingMarker *marker, + double search_pixel_x[5], double search_pixel_y[5]) +{ + int i; + float unified_coords[2]; + float pixel_coords[2]; + + /* Convert the corners into search space coordinates. */ + for (i = 0; i < 4; i++) { + marker_unified_to_search_pixel(frame_width, frame_height, marker, marker->pattern_corners[i], pixel_coords); + search_pixel_x[i] = pixel_coords[0]; + search_pixel_y[i] = pixel_coords[1]; + } + /* Convert the center position (aka "pos"); this is the origin */ + unified_coords[0] = 0.0; + unified_coords[1] = 0.0; + marker_unified_to_search_pixel(frame_width, frame_height, marker, unified_coords, pixel_coords); + + search_pixel_x[4] = pixel_coords[0]; + search_pixel_y[4] = pixel_coords[1]; +} + +/* Inverse of above. */ +static void set_marker_coords_from_tracking(int frame_width, int frame_height, MovieTrackingMarker *marker, + const double search_pixel_x[5], const double search_pixel_y[5]) +{ + int i; + float marker_unified[2]; + float search_pixel[2]; + + /* Convert the corners into search space coordinates. */ + for (i = 0; i < 4; i++) { + search_pixel[0] = search_pixel_x[i]; + search_pixel[1] = search_pixel_y[i]; + search_pixel_to_marker_unified(frame_width, frame_height, marker, search_pixel, marker->pattern_corners[i]); + } + + /* Convert the center position (aka "pos"); this is the origin */ + search_pixel[0] = search_pixel_x[4]; + search_pixel[1] = search_pixel_y[4]; + search_pixel_to_marker_unified(frame_width, frame_height, marker, search_pixel, marker_unified); + + /* If the tracker tracked nothing, then "marker_unified" would be zero. + * Otherwise, the entire patch shifted, and that delta should be applied to + * all the coordinates. + */ + for (i = 0; i < 4; i++) { + marker->pattern_corners[i][0] -= marker_unified[0]; + marker->pattern_corners[i][1] -= marker_unified[1]; + } + + marker->pos[0] += marker_unified[0]; + marker->pos[1] += marker_unified[1]; +} +#endif + /*********************** common functions *************************/ void BKE_tracking_init_settings(MovieTracking *tracking) @@ -81,11 +219,10 @@ void BKE_tracking_init_settings(MovieTracking *tracking) tracking->camera.pixel_aspect = 1.0f; tracking->camera.units = CAMERA_UNITS_MM; - tracking->settings.default_tracker = TRACKER_HYBRID; + tracking->settings.default_motion_model = TRACK_MOTION_MODEL_TRANSLATION; tracking->settings.default_minimum_correlation = 0.75; tracking->settings.default_pattern_size = 11; tracking->settings.default_search_size = 61; - tracking->settings.default_pyramid_levels = 2; tracking->settings.keyframe1 = 1; tracking->settings.keyframe2 = 30; tracking->settings.dist = 1; @@ -99,105 +236,68 @@ void BKE_tracking_init_settings(MovieTracking *tracking) BKE_tracking_new_object(tracking, "Camera"); } -void BKE_tracking_clamp_track(MovieTrackingTrack *track, int event) +void BKE_tracking_clamp_marker(MovieTrackingMarker *marker, int event) { int a; - float pat_min[2]; - float pat_max[2]; - float max_pyramid_level_factor = 1.0; - - if (track->tracker == TRACKER_KLT) { - max_pyramid_level_factor = 1 << (track->pyramid_levels - 1); - } - - /* sort */ - for (a = 0; a < 2; a++) { - if (track->pat_min[a] > track->pat_max[a]) - SWAP(float, track->pat_min[a], track->pat_max[a]); - - if (track->search_min[a] > track->search_max[a]) - SWAP(float, track->search_min[a], track->search_max[a]); - } + float pat_min[2], pat_max[2]; - /* compute the effective pattern size, which differs from the fine resolution - * pattern size for the pyramid KLT tracker */ - for (a = 0; a < 2; a++) { - pat_min[a] = max_pyramid_level_factor * track->pat_min[a]; - pat_max[a] = max_pyramid_level_factor * track->pat_max[a]; - } + BKE_tracking_marker_pattern_minmax(marker, pat_min, pat_max); if (event == CLAMP_PAT_DIM) { for (a = 0; a < 2; a++) { /* search shouldn't be resized smaller than pattern */ - track->search_min[a] = MIN2(pat_min[a], track->search_min[a]); - track->search_max[a] = MAX2(pat_max[a], track->search_max[a]); + marker->search_min[a] = MIN2(pat_min[a], marker->search_min[a]); + marker->search_max[a] = MAX2(pat_max[a], marker->search_max[a]); } } else if (event == CLAMP_PAT_POS) { float dim[2]; - sub_v2_v2v2(dim, track->pat_max, track->pat_min); + sub_v2_v2v2(dim, pat_max, pat_min); for (a = 0; a < 2; a++) { + int b; /* pattern shouldn't be moved outside of search */ - if (pat_min[a] < track->search_min[a]) { - track->pat_min[a] = track->search_min[a] - (pat_min[a] - track->pat_min[a]); - track->pat_max[a] = track->pat_min[a] + dim[a]; + if (pat_min[a] < marker->search_min[a]) { + for (b = 0; b < 4; b++) + marker->pattern_corners[b][a] += marker->search_min[a] - pat_min[a]; } - if (track->pat_max[a] > track->search_max[a]) { - track->pat_max[a] = track->search_max[a] - (pat_max[a] - track->pat_max[a]); - track->pat_min[a] = track->pat_max[a] - dim[a]; + if (pat_max[a] > marker->search_max[a]) { + for (b = 0; b < 4; b++) + marker->pattern_corners[b][a] -= pat_max[a] - marker->search_max[a]; } } } else if (event == CLAMP_SEARCH_DIM) { for (a = 0; a < 2; a++) { /* search shouldn't be resized smaller than pattern */ - track->search_min[a] = MIN2(pat_min[a], track->search_min[a]); - track->search_max[a] = MAX2(pat_max[a], track->search_max[a]); + marker->search_min[a] = MIN2(pat_min[a], marker->search_min[a]); + marker->search_max[a] = MAX2(pat_max[a], marker->search_max[a]); } } else if (event == CLAMP_SEARCH_POS) { float dim[2]; - sub_v2_v2v2(dim, track->search_max, track->search_min); + sub_v2_v2v2(dim, marker->search_max, marker->search_min); for (a = 0; a < 2; a++) { /* search shouldn't be moved inside pattern */ - if (track->search_min[a] > pat_min[a]) { - track->search_min[a] = pat_min[a]; - track->search_max[a] = track->search_min[a] + dim[a]; - } - if (track->search_max[a] < pat_max[a]) { - track->search_max[a] = pat_max[a]; - track->search_min[a] = track->search_max[a] - dim[a]; + if (marker->search_min[a] > pat_min[a]) { + marker->search_min[a] = pat_min[a]; + marker->search_max[a] = marker->search_min[a] + dim[a]; } - } - } - else if (event == CLAMP_PYRAMID_LEVELS || (event == CLAMP_SEARCH_DIM && track->tracker == TRACKER_KLT)) { - float dim[2]; - sub_v2_v2v2(dim, track->pat_max, track->pat_min); - { - float search_ratio = 2.3f * max_pyramid_level_factor; - - /* resize the search area to something sensible based - * on the number of pyramid levels */ - for (a = 0; a < 2; a++) { - track->search_min[a] = search_ratio * track->pat_min[a]; - track->search_max[a] = search_ratio * track->pat_max[a]; + if (marker->search_max[a] < pat_max[a]) { + marker->search_max[a] = pat_max[a]; + marker->search_min[a] = marker->search_max[a] - dim[a]; } } } - - /* marker's center should be in center of pattern */ - if (event == CLAMP_PAT_DIM || event == CLAMP_PAT_POS) { + else if (event == CLAMP_SEARCH_DIM) { float dim[2]; - - sub_v2_v2v2(dim, track->pat_max, track->pat_min); - + sub_v2_v2v2(dim, pat_max, pat_min); for (a = 0; a < 2; a++) { - track->pat_min[a] = -dim[a] / 2.0f; - track->pat_max[a] = dim[a] / 2.0f; + marker->search_min[a] = pat_min[a]; + marker->search_max[a] = pat_max[a]; } } } @@ -245,29 +345,32 @@ MovieTrackingTrack *BKE_tracking_add_track(MovieTracking *tracking, ListBase *tr track = MEM_callocN(sizeof(MovieTrackingTrack), "add_marker_exec track"); strcpy(track->name, "Track"); - track->tracker = settings->default_tracker; - track->pyramid_levels = settings->default_pyramid_levels; + track->motion_model = settings->default_motion_model; track->minimum_correlation = settings->default_minimum_correlation; track->margin = settings->default_margin; track->pattern_match = settings->default_pattern_match; track->frames_limit = settings->default_frames_limit; track->flag = settings->default_flag; + track->algorithm_flag = settings->default_algorithm_flag; memset(&marker, 0, sizeof(marker)); marker.pos[0] = x; marker.pos[1] = y; marker.framenr = framenr; - copy_v2_v2(track->pat_max, pat); - negate_v2_v2(track->pat_min, pat); + marker.pattern_corners[0][0] = -pat[0]; + marker.pattern_corners[0][1] = -pat[1]; - copy_v2_v2(track->search_max, search); - negate_v2_v2(track->search_min, search); + marker.pattern_corners[1][0] = pat[0]; + marker.pattern_corners[1][1] = -pat[1]; - BKE_tracking_insert_marker(track, &marker); + negate_v2_v2(marker.pattern_corners[2], marker.pattern_corners[0]); + negate_v2_v2(marker.pattern_corners[3], marker.pattern_corners[1]); - if (track->tracker == TRACKER_KLT) - BKE_tracking_clamp_track(track, CLAMP_PYRAMID_LEVELS); + copy_v2_v2(marker.search_max, search); + negate_v2_v2(marker.search_min, search); + + BKE_tracking_insert_marker(track, &marker); BLI_addtail(tracksbase, track); BKE_track_unique_name(tracksbase, track); @@ -337,6 +440,16 @@ void BKE_tracking_delete_marker(MovieTrackingTrack *track, int framenr) } } +void BKE_tracking_marker_pattern_minmax(MovieTrackingMarker *marker, float min[2], float max[2]) +{ + INIT_MINMAX2(min, max); + + DO_MINMAX2(marker->pattern_corners[0], min, max); + DO_MINMAX2(marker->pattern_corners[1], min, max); + DO_MINMAX2(marker->pattern_corners[2], min, max); + DO_MINMAX2(marker->pattern_corners[3], min, max); +} + MovieTrackingMarker *BKE_tracking_get_marker(MovieTrackingTrack *track, int framenr) { int a = track->markersnr - 1; @@ -527,7 +640,8 @@ void BKE_tracking_join_tracks(MovieTrackingTrack *dst_track, MovieTrackingTrack if ((dst_track->markers[b].flag & MARKER_DISABLED) == 0) { /* both tracks are enabled on this frame, so find the whole segment * on which tracks are intersecting and blend tracks using linear - * interpolation to prevent jumps */ + * interpolation to prevent jumps + */ MovieTrackingMarker *marker_a, *marker_b; int start_a = a, start_b = b, len = 0, frame = src_track->markers[a].framenr; @@ -827,7 +941,8 @@ static void tracks_map_merge(TracksMap *map, MovieTracking *tracking) /* duplicate currently operating tracks to temporary list. * this is needed to keep names in unique state and it's faster to change names - * of currently operating tracks (if needed) */ + * of currently operating tracks (if needed) + */ for (a = 0; a < map->num_tracks; a++) { int replace_sel = 0, replace_rot = 0; MovieTrackingTrack *new_track, *old; @@ -929,10 +1044,14 @@ static void tracks_map_free(TracksMap *map, void (*customdata_free) (void *custo typedef struct TrackContext { #ifdef WITH_LIBMV - float keyframed_pos[2]; + /* the reference marker and cutout search area */ + MovieTrackingMarker marker; - struct libmv_RegionTracker *region_tracker; - float *patch; /* keyframed patch */ + /* keyframed patch. This is the search area */ + float *search_area; + int search_area_height; + int search_area_width; + int framenr; #else int pad; #endif @@ -999,49 +1118,7 @@ MovieTrackingContext *BKE_tracking_context_new(MovieClip *clip, MovieClipUser *u if ((marker->flag & MARKER_DISABLED) == 0) { TrackContext track_context; - memset(&track_context, 0, sizeof(TrackContext)); - -#ifdef WITH_LIBMV - { - float patx = (int)((track->pat_max[0] - track->pat_min[0]) * width), - paty = (int)((track->pat_max[1] - track->pat_min[1]) * height); - - float search_size_x = (track->search_max[0] - track->search_min[0]) * width; - float search_size_y = (track->search_max[1] - track->search_min[1]) * height; - float pattern_size_x = (track->pat_max[0] - track->pat_min[0]) * width; - float pattern_size_y = (track->pat_max[1] - track->pat_min[1]) * height; - int wndx = (int)patx / 2, wndy = (int)paty / 2; - int half_wnd = MAX2(wndx, wndy); - - /* compute the maximum pyramid size */ - float search_to_pattern_ratio = MIN2(search_size_x, search_size_y) - / MAX2(pattern_size_x, pattern_size_y); - float log2_search_to_pattern_ratio = log(floor(search_to_pattern_ratio)) / M_LN2; - int max_pyramid_levels = floor(log2_search_to_pattern_ratio + 1); - - /* try to accommodate the user's choice of pyramid level in a way - * that doesn't cause the coarsest pyramid pattern to be larger - * than the search size */ - int level = MIN2(track->pyramid_levels, max_pyramid_levels); - - struct libmv_RegionTracker *region_tracker; - - if (track->tracker == TRACKER_KLT) { - region_tracker = libmv_pyramidRegionTrackerNew(100, level, half_wnd, - track->minimum_correlation); - } - else if (track->tracker == TRACKER_HYBRID) { - region_tracker = libmv_hybridRegionTrackerNew(100, half_wnd, track->minimum_correlation); - } - else if (track->tracker == TRACKER_SAD) { - region_tracker = libmv_bruteRegionTrackerNew(MAX2(wndx, wndy), track->minimum_correlation); - } - - track_context.region_tracker = region_tracker; - } -#endif - tracks_map_insert(context->tracks_map, track, &track_context); } } @@ -1059,7 +1136,8 @@ MovieTrackingContext *BKE_tracking_context_new(MovieClip *clip, MovieClipUser *u * would be used for images * - MCLIP_USE_PROXY_CUSTOM_DIR is needed because proxy/timecode files might * be stored in a different location - * ignore all the rest possible flags for now */ + * ignore all the rest possible flags for now + */ context->clip_flag = clip->flag & MCLIP_TIMECODE_FLAGS; context->user = *user; @@ -1077,11 +1155,8 @@ static void track_context_free(void *customdata) TrackContext *track_context = (TrackContext *)customdata; #if WITH_LIBMV - if (track_context->region_tracker) - libmv_regionTrackerDestroy(track_context->region_tracker); - - if (track_context->patch) - MEM_freeN(track_context->patch); + if (track_context->search_area) + MEM_freeN(track_context->search_area); #else (void) track_context; @@ -1100,7 +1175,8 @@ void BKE_tracking_context_free(MovieTrackingContext *context) /* zap channels from the imbuf that are disabled by the user. this can lead to * better tracks sometimes. however, instead of simply zeroing the channels - * out, do a partial grayscale conversion so the display is better. */ + * out, do a partial grayscale conversion so the display is better. + */ void BKE_tracking_disable_imbuf_channels(ImBuf *ibuf, int disable_red, int disable_green, int disable_blue, int grayscale) { @@ -1111,7 +1187,8 @@ void BKE_tracking_disable_imbuf_channels(ImBuf *ibuf, int disable_red, int disab return; /* If only some components are selected, it's important to rescale the result - * appropriately so that e.g. if only blue is selected, it's not zeroed out. */ + * appropriately so that e.g. if only blue is selected, it's not zeroed out. + */ scale = (disable_red ? 0.0f : 0.2126f) + (disable_green ? 0.0f : 0.7152f) + (disable_blue ? 0.0f : 0.0722f); @@ -1167,116 +1244,249 @@ static void disable_imbuf_channels(ImBuf *ibuf, MovieTrackingTrack *track, int g track->flag & TRACK_DISABLE_GREEN, track->flag & TRACK_DISABLE_BLUE, grayscale); } -static ImBuf *get_area_imbuf(ImBuf *ibuf, MovieTrackingTrack *track, MovieTrackingMarker *marker, - float min[2], float max[2], int margin, int anchored, float pos[2], int origin[2]) +ImBuf *BKE_tracking_sample_pattern_imbuf(int frame_width, int frame_height, + ImBuf *search_ibuf, MovieTrackingMarker *marker, + int num_samples_x, int num_samples_y, float pos[2]) { - ImBuf *tmpibuf; - int x, y; - int x1, y1, w, h; - float mpos[2]; + ImBuf *pattern_ibuf; + double src_pixel_x[5], src_pixel_y[5]; + double warped_position_x, warped_position_y; - copy_v2_v2(mpos, marker->pos); - if (anchored) - add_v2_v2(mpos, track->offset); + pattern_ibuf = IMB_allocImBuf(num_samples_x, num_samples_y, 32, IB_rectfloat); + pattern_ibuf->profile = IB_PROFILE_LINEAR_RGB; - if (pos) - zero_v2(pos); + if (!search_ibuf->rect_float) { + IMB_float_from_rect(search_ibuf); + } - x = mpos[0]*ibuf->x; - y = mpos[1]*ibuf->y; + get_marker_coords_for_tracking(frame_width, frame_height, marker, src_pixel_x, src_pixel_y); - w = (max[0] - min[0]) * ibuf->x; - h = (max[1] - min[1]) * ibuf->y; + libmv_samplePlanarPatch(search_ibuf->rect_float, search_ibuf->x, search_ibuf->y, 4, + src_pixel_x, src_pixel_y, num_samples_x, + num_samples_y, pattern_ibuf->rect_float, + &warped_position_x, &warped_position_y); - /* dimensions should be odd */ - w = w | 1; - h = h | 1; + if (pos) { + pos[0] = warped_position_x; + pos[1] = warped_position_y; + } - x1 = x - (int)(w * (-min[0] / (max[0] - min[0]))); - y1 = y - (int)(h * (-min[1] / (max[1] - min[1]))); + return pattern_ibuf; +} - if (ibuf->rect_float) - tmpibuf = IMB_allocImBuf(w + margin * 2, h + margin * 2, 32, IB_rectfloat); - else - tmpibuf = IMB_allocImBuf(w + margin * 2, h + margin * 2, 32, IB_rect); +ImBuf *BKE_tracking_get_pattern_imbuf(ImBuf *ibuf, MovieTrackingTrack *track, MovieTrackingMarker *marker, + int anchored, int disable_channels) +{ + ImBuf *pattern_ibuf, *search_ibuf; + float pat_min[2], pat_max[2]; + int num_samples_x, num_samples_y; + + BKE_tracking_marker_pattern_minmax(marker, pat_min, pat_max); + + num_samples_x = (pat_max[0] - pat_min[0]) * ibuf->x; + num_samples_y = (pat_max[1] - pat_min[1]) * ibuf->y; - tmpibuf->profile = ibuf->profile; + search_ibuf = BKE_tracking_get_search_imbuf(ibuf, track, marker, anchored, disable_channels); - IMB_rectcpy(tmpibuf, ibuf, 0, 0, x1 - margin, y1 - margin, w + margin * 2, h + margin * 2); + pattern_ibuf = BKE_tracking_sample_pattern_imbuf(ibuf->x, ibuf->y, search_ibuf, marker, + num_samples_x, num_samples_y, NULL); + + IMB_freeImBuf(search_ibuf); + + return pattern_ibuf; +} + +ImBuf *BKE_tracking_get_search_imbuf(ImBuf *ibuf, MovieTrackingTrack *track, MovieTrackingMarker *marker, + int anchored, int disable_channels) +{ + ImBuf *searchibuf; + int x, y, w, h; + float search_origin[2]; - if (pos != NULL) { - pos[0] = mpos[0] * ibuf->x - x1 + margin; - pos[1] = mpos[1] * ibuf->y - y1 + margin; + get_search_origin_frame_pixel(ibuf->x, ibuf->y, marker, search_origin); + + x = search_origin[0]; + y = search_origin[1]; + + if (anchored) { + x += track->offset[0] * ibuf->x; + y += track->offset[1] * ibuf->y; } - if (origin != NULL) { - origin[0] = x1 - margin; - origin[1] = y1 - margin; + w = (marker->search_max[0] - marker->search_min[0]) * ibuf->x; + h = (marker->search_max[1] - marker->search_min[1]) * ibuf->y; + + searchibuf = IMB_allocImBuf(w, h, 32, ibuf->rect_float ? IB_rectfloat : IB_rect); + searchibuf->profile = ibuf->profile; + + IMB_rectcpy(searchibuf, ibuf, 0, 0, x, y, w, h); + + if (disable_channels) { + if ((track->flag & TRACK_PREVIEW_GRAYSCALE) || + (track->flag & TRACK_DISABLE_RED) || + (track->flag & TRACK_DISABLE_GREEN) || + (track->flag & TRACK_DISABLE_BLUE)) + { + disable_imbuf_channels(searchibuf, track, TRUE); + } } - if ((track->flag & TRACK_PREVIEW_GRAYSCALE) || - (track->flag & TRACK_DISABLE_RED) || - (track->flag & TRACK_DISABLE_GREEN) || - (track->flag & TRACK_DISABLE_BLUE)) - { - disable_imbuf_channels(tmpibuf, track, TRUE /* grayscale */); + return searchibuf; +} + +static bGPDlayer *track_mask_gpencil_layer_get(MovieTrackingTrack *track) +{ + bGPDlayer *layer; + + if (!track->gpd) + return NULL; + + layer = track->gpd->layers.first; + + while (layer) { + if (layer->flag & GP_LAYER_ACTIVE) + return layer; + + layer = layer->next; } - return tmpibuf; + return NULL; } -ImBuf *BKE_tracking_get_pattern_imbuf(ImBuf *ibuf, MovieTrackingTrack *track, MovieTrackingMarker *marker, - int margin, int anchored, float pos[2], int origin[2]) +static void track_mask_gpencil_layer_rasterize(MovieTracking *tracking, MovieTrackingMarker *marker, + bGPDlayer *layer, ImBuf *ibuf, int width, int height) { - return get_area_imbuf(ibuf, track, marker, track->pat_min, track->pat_max, margin, anchored, pos, origin); + bGPDframe *frame = layer->frames.first; + float *mask; + int x, y; + float aspy = 1.0f / tracking->camera.pixel_aspect; + + mask = MEM_callocN(ibuf->x * ibuf->y * sizeof(float), "track mask"); + + while (frame) { + bGPDstroke *stroke = frame->strokes.first; + + while (stroke) { + bGPDspoint *stroke_points = stroke->points; + float *mask_points, *fp; + int i; + + if (stroke->flag & GP_STROKE_2DSPACE) { + fp = mask_points = MEM_callocN(2 * stroke->totpoints * sizeof(float), + "track mask rasterization points"); + + for (i = 0; i < stroke->totpoints; i++, fp += 2) { + fp[0] = stroke_points[i].x * width / ibuf->x - marker->search_min[0]; + fp[1] = stroke_points[i].y * height * aspy / ibuf->x - marker->search_min[1]; + } + + PLX_raskterize((float (*)[2])mask_points, stroke->totpoints, mask, ibuf->x, ibuf->y); + + MEM_freeN(mask_points); + } + + stroke = stroke->next; + } + + frame = frame->next; + } + + for (y = 0; y < ibuf->y; y++) { + for (x = 0; x < ibuf->x; x++) { + float *pixel = &ibuf->rect_float[4 * (y * ibuf->x + x)]; + float val = mask[y * ibuf->x + x]; + + pixel[0] = val; + pixel[1] = val; + pixel[2] = val; + pixel[3] = 1.0f; + } + } + + MEM_freeN(mask); + + IMB_rect_from_float(ibuf); } -ImBuf *BKE_tracking_get_search_imbuf(ImBuf *ibuf, MovieTrackingTrack *track, MovieTrackingMarker *marker, - int margin, int anchored, float pos[2], int origin[2]) +ImBuf *BKE_tracking_track_mask_get(MovieTracking *tracking, MovieTrackingTrack *track, MovieTrackingMarker *marker, + int width, int height) { - return get_area_imbuf(ibuf, track, marker, track->search_min, track->search_max, margin, anchored, pos, origin); + ImBuf *ibuf; + bGPDlayer *layer = track_mask_gpencil_layer_get(track); + int mask_width, mask_height; + + mask_width = (marker->search_max[0] - marker->search_min[0]) * width; + mask_height = (marker->search_max[1] - marker->search_min[1]) * height; + + ibuf = IMB_allocImBuf(mask_width, mask_height, 32, IB_rect | IB_rectfloat); + + if (layer) { + track_mask_gpencil_layer_rasterize(tracking, marker, layer, ibuf, width, height); + } + else { + float white[4] = {1.0f, 1.0f, 1.0f, 1.0f}; + IMB_rectfill(ibuf, white); + } + + return ibuf; } #ifdef WITH_LIBMV -static float *get_search_floatbuf(ImBuf *ibuf, MovieTrackingTrack *track, MovieTrackingMarker *marker, - int *width_r, int *height_r, float pos[2], int origin[2]) + +/* Convert from float and byte RGBA to grayscale. Supports different coefficients for RGB. */ +static void float_rgba_to_gray(const float *rgba, float *gray, int num_pixels, + float weight_red, float weight_green, float weight_blue) { - ImBuf *tmpibuf; - float *pixels, *fp; - int x, y, width, height; + int i; - width = (track->search_max[0] - track->search_min[0]) * ibuf->x; - height = (track->search_max[1] - track->search_min[1]) * ibuf->y; + for (i = 0; i < num_pixels; i++) { + const float *pixel = rgba + 4 * i; - tmpibuf = BKE_tracking_get_search_imbuf(ibuf, track, marker, 0, 0, pos, origin); - disable_imbuf_channels(tmpibuf, track, FALSE /* don't grayscale */); + gray[i] = weight_red * pixel[0] + weight_green * pixel[1] + weight_blue * pixel[2]; + } +} - *width_r = width; - *height_r = height; +static void uint8_rgba_to_float_gray(const unsigned char *rgba, float *gray, int num_pixels, + float weight_red, float weight_green, float weight_blue) +{ + int i; - fp = pixels = MEM_callocN(width * height * sizeof(float), "tracking floatBuf"); - for (y = 0; y < (int)height; y++) { - for (x = 0; x < (int)width; x++) { - int pixel = tmpibuf->x * y + x; + for (i = 0; i < num_pixels; i++) { + const unsigned char *pixel = rgba + i * 4; - if (tmpibuf->rect_float) { - float *rrgbf = tmpibuf->rect_float + pixel * 4; + *gray++ = (weight_red * pixel[0] + weight_green * pixel[1] + weight_blue * pixel[2]) / 255.0f; + } +} - *fp = 0.2126 * rrgbf[0] + 0.7152 * rrgbf[1] + 0.0722 * rrgbf[2]; - } - else { - unsigned char *rrgb = (unsigned char*)tmpibuf->rect + pixel * 4; +static float *get_search_floatbuf(ImBuf *ibuf, MovieTrackingTrack *track, MovieTrackingMarker *marker, + int *width_r, int *height_r) +{ + ImBuf *searchibuf; + float *gray_pixels; + int width, height; - *fp = (0.2126 * rrgb[0] + 0.7152 * rrgb[1] + 0.0722 * rrgb[2]) / 255.0f; - } + searchibuf = BKE_tracking_get_search_imbuf(ibuf, track, marker, FALSE, TRUE); - fp++; - } + width = searchibuf->x; + height = searchibuf->y; + + *width_r = searchibuf->x; + *height_r = searchibuf->y; + + gray_pixels = MEM_callocN(width * height * sizeof(float), "tracking floatBuf"); + + if (searchibuf->rect_float) { + float_rgba_to_gray(searchibuf->rect_float, gray_pixels, width * height, + 0.2126f, 0.7152f, 0.0722f); + } + else { + uint8_rgba_to_float_gray((unsigned char *)searchibuf->rect, gray_pixels, width * height, + 0.2126f, 0.7152f, 0.0722f); } - IMB_freeImBuf(tmpibuf); + IMB_freeImBuf(searchibuf); - return pixels; + return gray_pixels; } static unsigned char *get_ucharbuf(ImBuf *ibuf) @@ -1400,10 +1610,12 @@ void BKE_tracking_sync_user(MovieClipUser *user, MovieTrackingContext *context) int BKE_tracking_next(MovieTrackingContext *context) { - ImBuf *ibuf_new; + ImBuf *destination_ibuf; int curfra = BKE_movieclip_remap_scene_to_clip_frame(context->clip, context->user.framenr); int a, ok = FALSE, map_size; + int frame_width, frame_height; + map_size = tracks_map_size(context->tracks_map); /* nothing to track, avoid unneeded frames reading to save time and memory */ @@ -1415,28 +1627,31 @@ int BKE_tracking_next(MovieTrackingContext *context) else context->user.framenr++; - ibuf_new = BKE_movieclip_get_ibuf_flag(context->clip, &context->user, context->clip_flag, MOVIECLIP_CACHE_SKIP); - if (!ibuf_new) + destination_ibuf = BKE_movieclip_get_ibuf_flag(context->clip, &context->user, + context->clip_flag, MOVIECLIP_CACHE_SKIP); + if (!destination_ibuf) return FALSE; - #pragma omp parallel for private(a) shared(ibuf_new, ok) if (map_size>1) + frame_width = destination_ibuf->x; + frame_height = destination_ibuf->y; + + #pragma omp parallel for private(a) shared(destination_ibuf, ok) if (map_size>1) for (a = 0; a < map_size; a++) { TrackContext *track_context = NULL; MovieTrackingTrack *track; MovieTrackingMarker *marker; - tracks_map_get(context->tracks_map, a, &track, (void**)&track_context); + tracks_map_get(context->tracks_map, a, &track, (void **)&track_context); marker = BKE_tracking_exact_marker(track, curfra); if (marker && (marker->flag & MARKER_DISABLED) == 0) { #ifdef WITH_LIBMV - int width, height, origin[2], tracked = 0, need_readjust = 0; - float pos[2], margin[2], dim[2]; - double x1, y1, x2, y2; - ImBuf *ibuf = NULL; + int width, height, tracked = 0, need_readjust = 0; + float margin[2], dim[2], pat_min[2], pat_max[2]; MovieTrackingMarker marker_new, *marker_keyed; int onbound = FALSE, nextfra; + double dst_pixel_x[5], dst_pixel_y[5]; if (track->pattern_match == TRACK_MATCH_KEYFRAME) need_readjust = context->first_time; @@ -1449,11 +1664,12 @@ int BKE_tracking_next(MovieTrackingContext *context) nextfra = curfra + 1; /* margin from frame boundaries */ - sub_v2_v2v2(dim, track->pat_max, track->pat_min); + BKE_tracking_marker_pattern_minmax(marker, pat_min, pat_max); + sub_v2_v2v2(dim, pat_max, pat_min); margin[0] = margin[1] = MAX2(dim[0], dim[1]) / 2.0f; - margin[0] = MAX2(margin[0], (float)track->margin / ibuf_new->x); - margin[1] = MAX2(margin[1], (float)track->margin / ibuf_new->y); + margin[0] = MAX2(margin[0], (float)track->margin / destination_ibuf->x); + margin[1] = MAX2(margin[1], (float)track->margin / destination_ibuf->y); /* do not track markers which are too close to boundary */ if (marker->pos[0] < margin[0] || marker->pos[0] > 1.0f - margin[0] || @@ -1462,58 +1678,83 @@ int BKE_tracking_next(MovieTrackingContext *context) onbound = TRUE; } else { + /* to convert to the x/y split array format for libmv. */ + double src_pixel_x[5]; + double src_pixel_y[5]; + + /* settings for the tracker */ + struct libmv_trackRegionOptions options; + struct libmv_trackRegionResult result; + float *patch_new; if (need_readjust) { + ImBuf *reference_ibuf = NULL; /* calculate patch for keyframed position */ - ibuf = get_adjust_ibuf(context, track, marker, curfra, &marker_keyed); + reference_ibuf = get_adjust_ibuf(context, track, marker, curfra, &marker_keyed); + track_context->marker = *marker_keyed; - if (track_context->patch) - MEM_freeN(track_context->patch); + if (track_context->search_area) + MEM_freeN(track_context->search_area); - track_context->patch = get_search_floatbuf(ibuf, track, marker_keyed, &width, &height, - track_context->keyframed_pos, origin); + track_context->search_area = get_search_floatbuf(reference_ibuf, track, + marker_keyed, &width, &height); + track_context->search_area_height = height; + track_context->search_area_width = width; - IMB_freeImBuf(ibuf); + IMB_freeImBuf(reference_ibuf); } - patch_new = get_search_floatbuf(ibuf_new, track, marker, &width, &height, pos, origin); + /* for now track to the same search area dimension as marker has got for current frame + * will make all tracked markers in currently tracked segment have the same search area + * size, but it's quite close to what is actually needed + */ + patch_new = get_search_floatbuf(destination_ibuf, track, marker, &width, &height); + + /* Configure the tracker */ + options.motion_model = track->motion_model; - x1 = track_context->keyframed_pos[0]; - y1 = track_context->keyframed_pos[1]; + options.use_brute = + ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_BRUTE) != 0); - x2 = pos[0]; - y2 = pos[1]; + options.use_normalization = + ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_NORMALIZATION) != 0); - tracked = libmv_regionTrackerTrack(track_context->region_tracker, track_context->patch, patch_new, - width, height, x1, y1, &x2, &y2); + options.num_iterations = 50; + options.minimum_correlation = track->minimum_correlation; + options.sigma = 0.9; + /* Convert the marker corners and center into pixel coordinates in the search/destination images. */ + get_marker_coords_for_tracking(frame_width, frame_height, &track_context->marker, src_pixel_x, src_pixel_y); + get_marker_coords_for_tracking(frame_width, frame_height, marker, dst_pixel_x, dst_pixel_y); + + /* Run the tracker! */ + tracked = libmv_trackRegion(&options, + track_context->search_area, patch_new, + width, height, + src_pixel_x, src_pixel_y, + &result, + dst_pixel_x, dst_pixel_y); MEM_freeN(patch_new); } - if (tracked && !onbound && finite(x2) && finite(y2)) { + if (tracked && !onbound) { + memset(&marker_new, 0, sizeof(marker_new)); + marker_new = *marker; + set_marker_coords_from_tracking(frame_width, frame_height, &marker_new, dst_pixel_x, dst_pixel_y); + marker_new.flag |= MARKER_TRACKED; + marker_new.framenr = nextfra; + if (context->first_time) { #pragma omp critical { /* check if there's no keyframe/tracked markers before tracking marker. - * if so -- create disabled marker before currently tracking "segment" */ - put_disabled_marker(track, marker, !context->backwards, 0); + * if so -- create disabled marker before currently tracking "segment" + */ + put_disabled_marker(track, &marker_new, !context->backwards, 0); } } - memset(&marker_new, 0, sizeof(marker_new)); - - if (!onbound) { - marker_new.pos[0] = (origin[0] + x2) / ibuf_new->x; - marker_new.pos[1] = (origin[1] + y2) / ibuf_new->y; - } - else { - copy_v2_v2(marker_new.pos, marker->pos); - } - - marker_new.flag |= MARKER_TRACKED; - marker_new.framenr = nextfra; - #pragma omp critical { BKE_tracking_insert_marker(track, &marker_new); @@ -1542,7 +1783,7 @@ int BKE_tracking_next(MovieTrackingContext *context) } } - IMB_freeImBuf(ibuf_new); + IMB_freeImBuf(destination_ibuf); context->first_time = FALSE; context->frames++; @@ -2765,7 +3006,8 @@ ImBuf *BKE_tracking_stabilize(MovieTracking *tracking, int framenr, ImBuf *ibuf, if (tangle == 0.0f) { /* if angle is zero, then it's much faster to use rect copy - * but could be issues with subpixel precisions */ + * but could be issues with subpixel precisions + */ IMB_rectcpy(tmpibuf, ibuf, tloc[0] - (tscale - 1.0f) * width / 2.0f, tloc[1] - (tscale - 1.0f) * height / 2.0f, diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c index fccdcbef564..cd0bbb314eb 100644 --- a/source/blender/blenloader/intern/readfile.c +++ b/source/blender/blenloader/intern/readfile.c @@ -5362,6 +5362,7 @@ static void lib_link_screen(FileData *fd, Main *main) sclip->clip = newlibadr_us(fd, sc->id.lib, sclip->clip); sclip->mask = newlibadr_us(fd, sc->id.lib, sclip->mask); + sclip->scopes.track_search = NULL; sclip->scopes.track_preview = NULL; sclip->draw_context = NULL; sclip->scopes.ok = 0; @@ -7084,12 +7085,9 @@ static void do_versions(FileData *fd, Library *lib, Main *main) track = clip->tracking.tracks.first; while (track) { - if (track->pyramid_levels == 0) - track->pyramid_levels = 2; - if (track->minimum_correlation == 0.0f) track->minimum_correlation = 0.75f; - + track = track->next; } } @@ -7107,10 +7105,9 @@ static void do_versions(FileData *fd, Library *lib, Main *main) for (clip= main->movieclip.first; clip; clip= clip->id.next) { MovieTrackingSettings *settings= &clip->tracking.settings; - - if (settings->default_pyramid_levels == 0) { - settings->default_tracker= TRACKER_KLT; - settings->default_pyramid_levels = 2; + + if (settings->default_pattern_size == 0.0f) { + settings->default_motion_model = TRACK_MOTION_MODEL_TRANSLATION; settings->default_minimum_correlation = 0.75; settings->default_pattern_size = 11; settings->default_search_size = 51; @@ -7638,6 +7635,7 @@ static void do_versions(FileData *fd, Library *lib, Main *main) } } + if (main->versionfile < 263 || (main->versionfile == 263 && main->subversionfile < 8)) { /* set new deactivation values for game settings */ @@ -7702,6 +7700,46 @@ static void do_versions(FileData *fd, Library *lib, Main *main) } } + { + MovieClip *clip; + + for (clip = main->movieclip.first; clip; clip = clip->id.next) { + MovieTrackingTrack *track; + + track = clip->tracking.tracks.first; + while (track) { + int i; + + for (i = 0; i < track->markersnr; i++) { + MovieTrackingMarker *marker = &track->markers[i]; + + if (is_zero_v2(marker->pattern_corners[0]) && is_zero_v2(marker->pattern_corners[1]) && + is_zero_v2(marker->pattern_corners[3]) && is_zero_v2(marker->pattern_corners[3])) + { + marker->pattern_corners[0][0] = track->pat_min[0]; + marker->pattern_corners[0][1] = track->pat_min[1]; + + marker->pattern_corners[1][0] = track->pat_max[0]; + marker->pattern_corners[1][1] = track->pat_min[1]; + + marker->pattern_corners[2][0] = track->pat_max[0]; + marker->pattern_corners[2][1] = track->pat_max[1]; + + marker->pattern_corners[3][0] = track->pat_min[0]; + marker->pattern_corners[3][1] = track->pat_max[1]; + } + + if (is_zero_v2(marker->search_min) && is_zero_v2(marker->search_max)) { + copy_v2_v2(marker->search_min, track->search_min); + copy_v2_v2(marker->search_max, track->search_max); + } + } + + track = track->next; + } + } + } + /* WATCH IT!!!: pointers from libdata have not been converted yet here! */ /* WATCH IT 2!: Userdef struct init has to be in editors/interface/resources.c! */ diff --git a/source/blender/editors/interface/interface_draw.c b/source/blender/editors/interface/interface_draw.c index f6339d4456f..f368e7cf4c7 100644 --- a/source/blender/editors/interface/interface_draw.c +++ b/source/blender/editors/interface/interface_draw.c @@ -43,6 +43,7 @@ #include "BKE_colortools.h" #include "BKE_texture.h" +#include "BKE_tracking.h" #include "IMB_imbuf.h" @@ -1507,36 +1508,10 @@ void ui_draw_but_CURVE(ARegion *ar, uiBut *but, uiWidgetColors *wcol, rcti *rect fdrawbox(rect->xmin, rect->ymin, rect->xmax, rect->ymax); } -static ImBuf *scale_trackpreview_ibuf(ImBuf *ibuf, float track_pos[2], int width, float height, int margin) -{ - ImBuf *scaleibuf; - const float scalex = ((float)ibuf->x - 2 * margin) / width; - const float scaley = ((float)ibuf->y - 2 * margin) / height; - /* NOTE: 1.0f = 0.5f for integer coordinate coorrection (center of pixel vs. left bottom corner of bixel) - * and 0.5f for centering image in preview (cross is draving at exact center of widget so image - * should be shifted by half of pixel for correct centering) - sergey */ - float off_x = (int)track_pos[0] - track_pos[0] + 1.0f; - float off_y = (int)track_pos[1] - track_pos[1] + 1.0f; - int x, y; - - scaleibuf = IMB_allocImBuf(width, height, 32, IB_rect); - - for (y = 0; y < height; y++) { - for (x = 0; x < width; x++) { - float src_x = scalex * (x) + margin - off_x; - float src_y = scaley * (y) + margin - off_y; - - bicubic_interpolation(ibuf, scaleibuf, src_x, src_y, x, y); - } - } - - return scaleibuf; -} - void ui_draw_but_TRACKPREVIEW(ARegion *ar, uiBut *but, uiWidgetColors *UNUSED(wcol), rcti *recti) { rctf rect; - int ok = 0; + int ok = 0, width, height; GLint scissor[4]; MovieClipScopes *scopes = (MovieClipScopes *)but->poin; @@ -1545,6 +1520,9 @@ void ui_draw_but_TRACKPREVIEW(ARegion *ar, uiBut *but, uiWidgetColors *UNUSED(wc rect.ymin = (float)recti->ymin + SCOPE_RESIZE_PAD + 2; rect.ymax = (float)recti->ymax - 1; + width = rect.xmax - rect.xmin + 1; + height = rect.ymax - rect.ymin; + glEnable(GL_BLEND); glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); @@ -1562,40 +1540,53 @@ void ui_draw_but_TRACKPREVIEW(ARegion *ar, uiBut *but, uiWidgetColors *UNUSED(wc ok = 1; } - else if (scopes->track_preview) { - /* additional margin around image */ - /* NOTE: should be kept in sync with value from BKE_movieclip_update_scopes */ - const int margin = 3; - float zoomx, zoomy, track_pos[2], off_x, off_y; - int a, width, height; + else if ((scopes->track_search) && + ((!scopes->track_preview) || + (scopes->track_preview->x != width || scopes->track_preview->y != height))) + { + ImBuf *tmpibuf; + + if (scopes->track_preview) + IMB_freeImBuf(scopes->track_preview); + + tmpibuf = BKE_tracking_sample_pattern_imbuf(scopes->frame_width, scopes->frame_height, + scopes->track_search, &scopes->undist_marker, + width, height, scopes->track_pos); + + if (tmpibuf->rect_float) + IMB_rect_from_float(tmpibuf); + + // XXX: for debug only + // tmpibuf->ftype = PNG; + // IMB_saveiff(tmpibuf, "sample.png", IB_rect); + + if (tmpibuf->rect) + scopes->track_preview = tmpibuf; + else + IMB_freeImBuf(tmpibuf); + } + + if (!ok && scopes->track_preview) { + float track_pos[2]; + int a; ImBuf *drawibuf; glPushMatrix(); - track_pos[0] = scopes->track_pos[0] - margin; - track_pos[1] = scopes->track_pos[1] - margin; + track_pos[0] = scopes->track_pos[0]; + track_pos[1] = scopes->track_pos[1]; /* draw content of pattern area */ glScissor(ar->winrct.xmin + rect.xmin, ar->winrct.ymin + rect.ymin, scissor[2], scissor[3]); - width = rect.xmax - rect.xmin + 1; - height = rect.ymax - rect.ymin; - if (width > 0 && height > 0) { - zoomx = (float)width / (scopes->track_preview->x - 2 * margin); - zoomy = (float)height / (scopes->track_preview->y - 2 * margin); - - off_x = ((int)track_pos[0] - track_pos[0] + 0.5f) * zoomx; - off_y = ((int)track_pos[1] - track_pos[1] + 0.5f) * zoomy; - - drawibuf = scale_trackpreview_ibuf(scopes->track_preview, track_pos, width, height, margin); + drawibuf = scopes->track_preview; glaDrawPixelsSafe(rect.xmin, rect.ymin + 1, drawibuf->x, drawibuf->y, drawibuf->x, GL_RGBA, GL_UNSIGNED_BYTE, drawibuf->rect); - IMB_freeImBuf(drawibuf); /* draw cross for pizel position */ - glTranslatef(off_x + rect.xmin + track_pos[0] * zoomx, off_y + rect.ymin + track_pos[1] * zoomy, 0.f); + glTranslatef(rect.xmin + track_pos[0], rect.ymin + track_pos[1], 0.f); glScissor(ar->winrct.xmin + rect.xmin, ar->winrct.ymin + rect.ymin, rect.xmax - rect.xmin, diff --git a/source/blender/editors/space_clip/clip_buttons.c b/source/blender/editors/space_clip/clip_buttons.c index e69e1f15b6e..ca2ae6e8461 100644 --- a/source/blender/editors/space_clip/clip_buttons.c +++ b/source/blender/editors/space_clip/clip_buttons.c @@ -186,12 +186,13 @@ typedef struct { MovieClip *clip; MovieClipUser *user; /* user of clip */ MovieTrackingTrack *track; + MovieTrackingMarker *marker; int framenr; /* current frame number */ float marker_pos[2]; /* position of marker in pixel coords */ - float track_pat[2]; /* position and dimensions of marker pattern in pixel coords */ + float marker_pat[2]; /* position and dimensions of marker pattern in pixel coords */ float track_offset[2]; /* offset of "parenting" point */ - float track_search_pos[2], track_search[2]; /* position and dimensions of marker search in pixel coords */ + float marker_search_pos[2], marker_search[2]; /* position and dimensions of marker search in pixel coords */ int marker_flag; /* marker's flags */ } MarkerUpdateCb; @@ -238,60 +239,63 @@ static void marker_block_handler(bContext *C, void *arg_cb, int event) ok = TRUE; } else if (event == B_MARKER_PAT_DIM) { - float dim[2], pat_dim[2]; + float dim[2], pat_dim[2], pat_min[2], pat_max[2]; + float scale_x, scale_y; + int a; - sub_v2_v2v2(pat_dim, cb->track->pat_max, cb->track->pat_min); + BKE_tracking_marker_pattern_minmax(cb->marker, pat_min, pat_max); - dim[0] = cb->track_pat[0] / width; - dim[1] = cb->track_pat[1] / height; + sub_v2_v2v2(pat_dim, pat_max, pat_min); - sub_v2_v2(dim, pat_dim); - mul_v2_fl(dim, 0.5f); + dim[0] = cb->marker_pat[0] / width; + dim[1] = cb->marker_pat[1] / height; - cb->track->pat_min[0] -= dim[0]; - cb->track->pat_min[1] -= dim[1]; + scale_x = dim[0] / pat_dim[0]; + scale_y = dim[1] / pat_dim[1]; - cb->track->pat_max[0] += dim[0]; - cb->track->pat_max[1] += dim[1]; + for (a = 0; a < 4; a++) { + cb->marker->pattern_corners[a][0] *= scale_x; + cb->marker->pattern_corners[a][1] *= scale_y; + } - BKE_tracking_clamp_track(cb->track, CLAMP_PAT_DIM); + BKE_tracking_clamp_marker(cb->marker, CLAMP_PAT_DIM); ok = TRUE; } else if (event == B_MARKER_SEARCH_POS) { float delta[2], side[2]; - sub_v2_v2v2(side, cb->track->search_max, cb->track->search_min); + sub_v2_v2v2(side, cb->marker->search_max, cb->marker->search_min); mul_v2_fl(side, 0.5f); - delta[0] = cb->track_search_pos[0] / width; - delta[1] = cb->track_search_pos[1] / height; + delta[0] = cb->marker_search_pos[0] / width; + delta[1] = cb->marker_search_pos[1] / height; - sub_v2_v2v2(cb->track->search_min, delta, side); - add_v2_v2v2(cb->track->search_max, delta, side); + sub_v2_v2v2(cb->marker->search_min, delta, side); + add_v2_v2v2(cb->marker->search_max, delta, side); - BKE_tracking_clamp_track(cb->track, CLAMP_SEARCH_POS); + BKE_tracking_clamp_marker(cb->marker, CLAMP_SEARCH_POS); ok = TRUE; } else if (event == B_MARKER_SEARCH_DIM) { float dim[2], search_dim[2]; - sub_v2_v2v2(search_dim, cb->track->search_max, cb->track->search_min); + sub_v2_v2v2(search_dim, cb->marker->search_max, cb->marker->search_min); - dim[0] = cb->track_search[0] / width; - dim[1] = cb->track_search[1] / height; + dim[0] = cb->marker_search[0] / width; + dim[1] = cb->marker_search[1] / height; sub_v2_v2(dim, search_dim); mul_v2_fl(dim, 0.5f); - cb->track->search_min[0] -= dim[0]; - cb->track->search_min[1] -= dim[1]; + cb->marker->search_min[0] -= dim[0]; + cb->marker->search_min[1] -= dim[1]; - cb->track->search_max[0] += dim[0]; - cb->track->search_max[1] += dim[1]; + cb->marker->search_max[0] += dim[0]; + cb->marker->search_max[1] += dim[1]; - BKE_tracking_clamp_track(cb->track, CLAMP_SEARCH_DIM); + BKE_tracking_clamp_marker(cb->marker, CLAMP_SEARCH_DIM); ok = TRUE; } @@ -337,6 +341,7 @@ void uiTemplateMarker(uiLayout *layout, PointerRNA *ptr, const char *propname, P MovieTrackingMarker *marker; MarkerUpdateCb *cb; const char *tip; + float pat_min[2], pat_max[2]; if (!ptr->data) return; @@ -366,6 +371,7 @@ void uiTemplateMarker(uiLayout *layout, PointerRNA *ptr, const char *propname, P cb->clip = clip; cb->user = user; cb->track = track; + cb->marker = marker; cb->marker_flag = marker->flag; cb->framenr = user->framenr; @@ -383,7 +389,7 @@ void uiTemplateMarker(uiLayout *layout, PointerRNA *ptr, const char *propname, P } else { int width, height, step, digits; - float pat_dim[2], pat_pos[2], search_dim[2], search_pos[2]; + float pat_dim[2], search_dim[2], search_pos[2]; uiLayout *col; BKE_movieclip_get_size(clip, user, &width, &height); @@ -399,19 +405,18 @@ void uiTemplateMarker(uiLayout *layout, PointerRNA *ptr, const char *propname, P step = 100; digits = 2; - sub_v2_v2v2(pat_dim, track->pat_max, track->pat_min); - sub_v2_v2v2(search_dim, track->search_max, track->search_min); + BKE_tracking_marker_pattern_minmax(marker, pat_min, pat_max); - add_v2_v2v2(search_pos, track->search_max, track->search_min); - mul_v2_fl(search_pos, 0.5); + sub_v2_v2v2(pat_dim, pat_max, pat_min); + sub_v2_v2v2(search_dim, marker->search_max, marker->search_min); - add_v2_v2v2(pat_pos, track->pat_max, track->pat_min); - mul_v2_fl(pat_pos, 0.5); + add_v2_v2v2(search_pos, marker->search_max, marker->search_min); + mul_v2_fl(search_pos, 0.5); to_pixel_space(cb->marker_pos, marker->pos, width, height); - to_pixel_space(cb->track_pat, pat_dim, width, height); - to_pixel_space(cb->track_search, search_dim, width, height); - to_pixel_space(cb->track_search_pos, search_pos, width, height); + to_pixel_space(cb->marker_pat, pat_dim, width, height); + to_pixel_space(cb->marker_search, search_dim, width, height); + to_pixel_space(cb->marker_search_pos, search_pos, width, height); to_pixel_space(cb->track_offset, track->offset, width, height); cb->marker_flag = marker->flag; @@ -447,19 +452,19 @@ void uiTemplateMarker(uiLayout *layout, PointerRNA *ptr, const char *propname, P -10*height, 10.0*height, step, digits, "Y-offset to parenting point"); uiDefBut(block, LABEL, 0, "Pattern Area:", 0, 114, 300, 19, NULL, 0, 0, 0, 0, ""); - uiDefButF(block, NUM, B_MARKER_PAT_DIM, "Width:", 10, 95, 300, 19, &cb->track_pat[0], 3.0f, + uiDefButF(block, NUM, B_MARKER_PAT_DIM, "Width:", 10, 95, 300, 19, &cb->marker_pat[0], 3.0f, 10.0*width, step, digits, "Width of marker's pattern in screen coordinates"); - uiDefButF(block, NUM, B_MARKER_PAT_DIM, "Height:", 10, 76, 300, 19, &cb->track_pat[1], 3.0f, + uiDefButF(block, NUM, B_MARKER_PAT_DIM, "Height:", 10, 76, 300, 19, &cb->marker_pat[1], 3.0f, 10.0*height, step, digits, "Height of marker's pattern in screen coordinates"); uiDefBut(block, LABEL, 0, "Search Area:", 0, 57, 300, 19, NULL, 0, 0, 0, 0, ""); - uiDefButF(block, NUM, B_MARKER_SEARCH_POS, "X:", 10, 38, 145, 19, &cb->track_search_pos[0], + uiDefButF(block, NUM, B_MARKER_SEARCH_POS, "X:", 10, 38, 145, 19, &cb->marker_search_pos[0], -width, width, step, digits, "X-position of search at frame relative to marker's position"); - uiDefButF(block, NUM, B_MARKER_SEARCH_POS, "Y:", 165, 38, 145, 19, &cb->track_search_pos[1], + uiDefButF(block, NUM, B_MARKER_SEARCH_POS, "Y:", 165, 38, 145, 19, &cb->marker_search_pos[1], -height, height, step, digits, "X-position of search at frame relative to marker's position"); - uiDefButF(block, NUM, B_MARKER_SEARCH_DIM, "Width:", 10, 19, 300, 19, &cb->track_search[0], 3.0f, + uiDefButF(block, NUM, B_MARKER_SEARCH_DIM, "Width:", 10, 19, 300, 19, &cb->marker_search[0], 3.0f, 10.0*width, step, digits, "Width of marker's search in screen soordinates"); - uiDefButF(block, NUM, B_MARKER_SEARCH_DIM, "Height:", 10, 0, 300, 19, &cb->track_search[1], 3.0f, + uiDefButF(block, NUM, B_MARKER_SEARCH_DIM, "Height:", 10, 0, 300, 19, &cb->marker_search[1], 3.0f, 10.0*height, step, digits, "Height of marker's search in screen soordinates"); uiBlockEndAlign(block); diff --git a/source/blender/editors/space_clip/clip_draw.c b/source/blender/editors/space_clip/clip_draw.c index 93a32fb06fc..9332413b33b 100644 --- a/source/blender/editors/space_clip/clip_draw.c +++ b/source/blender/editors/space_clip/clip_draw.c @@ -456,14 +456,17 @@ static void draw_marker_outline(SpaceClip *sc, MovieTrackingTrack *track, MovieT if ((marker->flag & MARKER_DISABLED) == 0) { float pos[2]; - rctf r; + float p[2]; - BLI_init_rctf(&r, track->pat_min[0], track->pat_max[0], track->pat_min[1], track->pat_max[1]); add_v2_v2v2(pos, marker->pos, track->offset); ED_clip_point_undistorted_pos(sc, pos, pos); - if (BLI_in_rctf(&r, pos[0] - marker_pos[0], pos[1] - marker_pos[1])) { + sub_v2_v2v2(p, pos, marker_pos); + + if (isect_point_quad_v2(p, marker->pattern_corners[0], marker->pattern_corners[1], + marker->pattern_corners[2], marker->pattern_corners[3])) + { if (tiny) glPointSize(3.0f); else glPointSize(4.0f); glBegin(GL_POINTS); @@ -499,10 +502,10 @@ static void draw_marker_outline(SpaceClip *sc, MovieTrackingTrack *track, MovieT if (sc->flag & SC_SHOW_MARKER_PATTERN) { glBegin(GL_LINE_LOOP); - glVertex2f(track->pat_min[0], track->pat_min[1]); - glVertex2f(track->pat_max[0], track->pat_min[1]); - glVertex2f(track->pat_max[0], track->pat_max[1]); - glVertex2f(track->pat_min[0], track->pat_max[1]); + glVertex2fv(marker->pattern_corners[0]); + glVertex2fv(marker->pattern_corners[1]); + glVertex2fv(marker->pattern_corners[2]); + glVertex2fv(marker->pattern_corners[3]); glEnd(); } @@ -510,10 +513,10 @@ static void draw_marker_outline(SpaceClip *sc, MovieTrackingTrack *track, MovieT ((marker->flag & MARKER_DISABLED) == 0 || (sc->flag & SC_SHOW_MARKER_PATTERN) == 0); if (sc->flag & SC_SHOW_MARKER_SEARCH && show_search) { glBegin(GL_LINE_LOOP); - glVertex2f(track->search_min[0], track->search_min[1]); - glVertex2f(track->search_max[0], track->search_min[1]); - glVertex2f(track->search_max[0], track->search_max[1]); - glVertex2f(track->search_min[0], track->search_max[1]); + glVertex2f(marker->search_min[0], marker->search_min[1]); + glVertex2f(marker->search_max[0], marker->search_min[1]); + glVertex2f(marker->search_max[0], marker->search_max[1]); + glVertex2f(marker->search_min[0], marker->search_max[1]); glEnd(); } glPopMatrix(); @@ -556,8 +559,7 @@ static void draw_marker_areas(SpaceClip *sc, MovieTrackingTrack *track, MovieTra /* marker position and offset position */ if ((track->flag & SELECT) == sel && (marker->flag & MARKER_DISABLED) == 0) { - float pos[2]; - rctf r; + float pos[2], p[2]; if (track->flag & TRACK_LOCKED) { if (act) @@ -574,11 +576,14 @@ static void draw_marker_areas(SpaceClip *sc, MovieTrackingTrack *track, MovieTra glColor3fv(col); } - BLI_init_rctf(&r, track->pat_min[0], track->pat_max[0], track->pat_min[1], track->pat_max[1]); add_v2_v2v2(pos, marker->pos, track->offset); ED_clip_point_undistorted_pos(sc, pos, pos); - if (BLI_in_rctf(&r, pos[0] - marker_pos[0], pos[1] - marker_pos[1])) { + sub_v2_v2v2(p, pos, marker_pos); + + if (isect_point_quad_v2(p, marker->pattern_corners[0], marker->pattern_corners[1], + marker->pattern_corners[2], marker->pattern_corners[3])) + { if (!tiny) glPointSize(2.0f); @@ -651,10 +656,10 @@ static void draw_marker_areas(SpaceClip *sc, MovieTrackingTrack *track, MovieTra } glBegin(GL_LINE_LOOP); - glVertex2f(track->pat_min[0], track->pat_min[1]); - glVertex2f(track->pat_max[0], track->pat_min[1]); - glVertex2f(track->pat_max[0], track->pat_max[1]); - glVertex2f(track->pat_min[0], track->pat_max[1]); + glVertex2fv(marker->pattern_corners[0]); + glVertex2fv(marker->pattern_corners[1]); + glVertex2fv(marker->pattern_corners[2]); + glVertex2fv(marker->pattern_corners[3]); glEnd(); } @@ -684,70 +689,82 @@ static void draw_marker_areas(SpaceClip *sc, MovieTrackingTrack *track, MovieTra } glBegin(GL_LINE_LOOP); - glVertex2f(track->search_min[0], track->search_min[1]); - glVertex2f(track->search_max[0], track->search_min[1]); - glVertex2f(track->search_max[0], track->search_max[1]); - glVertex2f(track->search_min[0], track->search_max[1]); + glVertex2f(marker->search_min[0], marker->search_min[1]); + glVertex2f(marker->search_max[0], marker->search_min[1]); + glVertex2f(marker->search_max[0], marker->search_max[1]); + glVertex2f(marker->search_min[0], marker->search_max[1]); glEnd(); } - /* pyramid */ - if (sel && TRACK_VIEW_SELECTED(sc, track) && - (track->tracker == TRACKER_KLT) && - (marker->flag & MARKER_DISABLED) == 0) - { - if (track->flag & TRACK_LOCKED) { - if (act) - UI_ThemeColor(TH_ACT_MARKER); - else if (track->pat_flag & SELECT) - UI_ThemeColorShade(TH_LOCK_MARKER, 64); - else UI_ThemeColor(TH_LOCK_MARKER); - } - else if (marker->flag & MARKER_DISABLED) { - if (act) - UI_ThemeColor(TH_ACT_MARKER); - else if (track->pat_flag & SELECT) - UI_ThemeColorShade(TH_DIS_MARKER, 128); - else UI_ThemeColor(TH_DIS_MARKER); - } - else { - if (track->pat_flag & SELECT) - glColor3fv(scol); - else - glColor3fv(col); - } - - { - int i = 0; - glPushMatrix(); - glEnable(GL_LINE_STIPPLE); - for (i = 1; i < track->pyramid_levels; ++i) { - glScalef(2.0f, 2.0f, 1.0); - } - /* only draw a pattern for the coarsest level */ - glBegin(GL_LINE_LOOP); - glVertex2f(track->pat_min[0], track->pat_min[1]); - glVertex2f(track->pat_max[0], track->pat_min[1]); - glVertex2f(track->pat_max[0], track->pat_max[1]); - glVertex2f(track->pat_min[0], track->pat_max[1]); - glEnd(); - glDisable(GL_LINE_STIPPLE); - glPopMatrix(); - } - } - if (tiny) glDisable(GL_LINE_STIPPLE); glPopMatrix(); } +static float get_shortest_pattern_side(MovieTrackingMarker *marker) +{ + int i, next; + float len = FLT_MAX; + + for (i = 0; i < 4; i++) { + float cur_len; + + next = (i + 1) % 4; + + cur_len = len_v2v2(marker->pattern_corners[i], marker->pattern_corners[next]); + + len = MIN2(cur_len, len); + } + + return len; +} + +static void draw_marker_slide_square(float x, float y, float dx, float dy, int outline, float px[2]) +{ + float tdx, tdy; + + tdx = dx; + tdy = dy; + + if (outline) { + tdx += px[0]; + tdy += px[1]; + } + + glBegin(GL_QUADS); + glVertex3f(x - tdx, y + tdy, 0.0f); + glVertex3f(x + tdx, y + tdy, 0.0f); + glVertex3f(x + tdx, y - tdy, 0.0f); + glVertex3f(x - tdx, y - tdy, 0.0f); + glEnd(); +} + +static void draw_marker_slide_triangle(float x, float y, float dx, float dy, int outline, float px[2]) +{ + float tdx, tdy; + + tdx = dx * 2.0f; + tdy = dy * 2.0f; + + if (outline) { + tdx += px[0]; + tdy += px[1]; + } + + glBegin(GL_TRIANGLES); + glVertex3f(x, y, 0.0f); + glVertex3f(x - tdx, y, 0.0f); + glVertex3f(x, y + tdy, 0.0f); + glEnd(); +} + static void draw_marker_slide_zones(SpaceClip *sc, MovieTrackingTrack *track, MovieTrackingMarker *marker, float marker_pos[2], int outline, int sel, int act, int width, int height) { - float x, y, dx, dy, patdx, patdy, searchdx, searchdy, tdx, tdy; + float dx, dy, patdx, patdy, searchdx, searchdy; int tiny = sc->flag & SC_SHOW_TINY_MARKER; - float col[3], scol[3], px[2]; + float col[3], scol[3], px[2], side; if ((tiny && outline) || (marker->flag & MARKER_DISABLED)) return; @@ -768,11 +785,12 @@ static void draw_marker_slide_zones(SpaceClip *sc, MovieTrackingTrack *track, Mo dx = 6.0f / width / sc->zoom; dy = 6.0f / height / sc->zoom; - patdx = MIN2(dx * 2.0f / 3.0f, (track->pat_max[0] - track->pat_min[0]) / 6.0f); - patdy = MIN2(dy * 2.0f / 3.0f, (track->pat_max[1] - track->pat_min[1]) / 6.0f); + side = get_shortest_pattern_side(marker); + patdx = MIN2(dx * 2.0f / 3.0f, side / 6.0f); + patdy = MIN2(dy * 2.0f / 3.0f, side * width / height / 6.0f); - searchdx = MIN2(dx, (track->search_max[0] - track->search_min[0]) / 6.0f); - searchdy = MIN2(dy, (track->search_max[1] - track->search_min[1]) / 6.0f); + searchdx = MIN2(dx, (marker->search_max[0] - marker->search_min[0]) / 6.0f); + searchdy = MIN2(dy, (marker->search_max[1] - marker->search_min[1]) / 6.0f); px[0] = 1.0f / sc->zoom / width / sc->scale; px[1] = 1.0f / sc->zoom / height / sc->scale; @@ -786,41 +804,10 @@ static void draw_marker_slide_zones(SpaceClip *sc, MovieTrackingTrack *track, Mo } /* search offset square */ - x = track->search_min[0]; - y = track->search_max[1]; - - tdx = searchdx; - tdy = searchdy; - - if (outline) { - tdx += px[0]; - tdy += px[1]; - } - - glBegin(GL_QUADS); - glVertex3f(x - tdx, y + tdy, 0); - glVertex3f(x + tdx, y + tdy, 0); - glVertex3f(x + tdx, y - tdy, 0); - glVertex3f(x - tdx, y - tdy, 0); - glEnd(); + draw_marker_slide_square(marker->search_min[0], marker->search_max[1], searchdx, searchdy, outline, px); /* search re-sizing triangle */ - x = track->search_max[0]; - y = track->search_min[1]; - - tdx = searchdx * 2.0f; - tdy = searchdy * 2.0f; - - if (outline) { - tdx += px[0]; - tdy += px[1]; - } - - glBegin(GL_TRIANGLES); - glVertex3f(x, y, 0); - glVertex3f(x - tdx, y, 0); - glVertex3f(x, y + tdy, 0); - glEnd(); + draw_marker_slide_triangle(marker->search_max[0], marker->search_min[1], searchdx, searchdy, outline, px); } if ((sc->flag & SC_SHOW_MARKER_PATTERN) && ((track->pat_flag & SELECT) == sel || outline)) { @@ -831,42 +818,26 @@ static void draw_marker_slide_zones(SpaceClip *sc, MovieTrackingTrack *track, Mo glColor3fv(col); } - /* pattern offset square */ - x = track->pat_min[0]; - y = track->pat_max[1]; + /* XXX: need to be real check if affine tracking is enabled, but for now not + * sure how to do this, so assume affine tracker is always enabled */ + if (TRUE) { + int i; - tdx = patdx; - tdy = patdy; - - if (outline) { - tdx += px[0]; - tdy += px[1]; + /* pattern's corners sliding squares */ + for (i = 0; i < 4; i++) { + draw_marker_slide_square(marker->pattern_corners[i][0], marker->pattern_corners[i][1], + patdx / 1.5f, patdy / 1.5f, outline, px); + } } + else { + /* pattern offset square */ + draw_marker_slide_square(marker->pattern_corners[3][0], marker->pattern_corners[3][1], + patdx, patdy, outline, px); - glBegin(GL_QUADS); - glVertex3f(x - tdx, y + tdy, 0); - glVertex3f(x + tdx, y + tdy, 0); - glVertex3f(x + tdx, y - tdy, 0); - glVertex3f(x - tdx, y - tdy, 0); - glEnd(); - - /* pattern re-sizing triangle */ - x = track->pat_max[0]; - y = track->pat_min[1]; - - tdx = patdx*2.0f; - tdy = patdy*2.0f; - - if (outline) { - tdx += px[0]; - tdy += px[1]; + /* pattern re-sizing triangle */ + draw_marker_slide_triangle(marker->pattern_corners[1][0], marker->pattern_corners[1][1], + patdx, patdy, outline, px); } - - glBegin(GL_TRIANGLES); - glVertex3f(x, y, 0); - glVertex3f(x - tdx, y, 0); - glVertex3f(x, y + tdy, 0); - glEnd(); } glPopMatrix(); @@ -905,12 +876,15 @@ static void draw_marker_texts(SpaceClip *sc, MovieTrackingTrack *track, MovieTra if ((sc->flag & SC_SHOW_MARKER_SEARCH) && ((marker->flag & MARKER_DISABLED) == 0 || (sc->flag & SC_SHOW_MARKER_PATTERN) == 0)) { - dx = track->search_min[0]; - dy = track->search_min[1]; + dx = marker->search_min[0]; + dy = marker->search_min[1]; } else if (sc->flag & SC_SHOW_MARKER_PATTERN) { - dx = track->pat_min[0]; - dy = track->pat_min[1]; + float pat_min[2], pat_max[2]; + + BKE_tracking_marker_pattern_minmax(marker, pat_min, pat_max); + dx = pat_min[0]; + dy = pat_min[1]; } pos[0] = (marker_pos[0] + dx) * width; diff --git a/source/blender/editors/space_clip/space_clip.c b/source/blender/editors/space_clip/space_clip.c index a49319abd20..54724881e37 100644 --- a/source/blender/editors/space_clip/space_clip.c +++ b/source/blender/editors/space_clip/space_clip.c @@ -306,6 +306,9 @@ static void clip_free(SpaceLink *sl) if (sc->scopes.track_preview) IMB_freeImBuf(sc->scopes.track_preview); + if (sc->scopes.track_search) + IMB_freeImBuf(sc->scopes.track_search); + ED_space_clip_free_texture_buffer(sc); } @@ -323,6 +326,7 @@ static SpaceLink *clip_duplicate(SpaceLink *sl) SpaceClip *scn = MEM_dupallocN(sl); /* clear or remove stuff from old */ + scn->scopes.track_search = NULL; scn->scopes.track_preview = NULL; scn->scopes.ok = FALSE; scn->draw_context = NULL; diff --git a/source/blender/editors/space_clip/tracking_ops.c b/source/blender/editors/space_clip/tracking_ops.c index 08fc84d193c..f6e9622f0a5 100644 --- a/source/blender/editors/space_clip/tracking_ops.c +++ b/source/blender/editors/space_clip/tracking_ops.c @@ -79,6 +79,8 @@ #include "clip_intern.h" // own include +static float dist_to_crns(float co[2], float pos[2], float crns[4][2]); + /********************** add marker operator *********************/ static void add_marker(SpaceClip *sc, float x, float y) @@ -260,15 +262,16 @@ typedef struct { int mval[2]; int width, height; - float *min, *max, *pos, *offset; - float smin[2], smax[2], spos[2], soff[2]; + float *min, *max, *pos, *offset, (*corners)[2]; + float smin[2], smax[2], spos[2], soff[2], scorners[4][2]; float (*smarkers)[2]; - int lock, accurate; + int lock, accurate, scale; } SlideMarkerData; static SlideMarkerData *create_slide_marker_data(SpaceClip *sc, MovieTrackingTrack *track, - MovieTrackingMarker *marker, wmEvent *event, int area, int action, int width, int height) + MovieTrackingMarker *marker, wmEvent *event, + int area, int corner, int action, int width, int height) { SlideMarkerData *data = MEM_callocN(sizeof(SlideMarkerData), "slide marker data"); int framenr = ED_space_clip_clip_framenr(sc); @@ -288,10 +291,9 @@ static SlideMarkerData *create_slide_marker_data(SpaceClip *sc, MovieTrackingTra } else if (area == TRACK_AREA_PAT) { if (action == SLIDE_ACTION_SIZE) { - data->min = track->pat_min; - data->max = track->pat_max; + data->corners = marker->pattern_corners; } - else { + else if (action == SLIDE_ACTION_OFFSET) { int a; data->pos = marker->pos; @@ -303,15 +305,28 @@ static SlideMarkerData *create_slide_marker_data(SpaceClip *sc, MovieTrackingTra for (a = 0; a < track->markersnr; a++) copy_v2_v2(data->smarkers[a], track->markers[a].pos); } + else if (action == SLIDE_ACTION_POS) { + data->corners = marker->pattern_corners; + data->pos = marker->pattern_corners[corner]; + + copy_v2_v2(data->spos, data->pos); + } } else if (area == TRACK_AREA_SEARCH) { - data->min = track->search_min; - data->max = track->search_max; + data->min = marker->search_min; + data->max = marker->search_max; } - if (area == TRACK_AREA_SEARCH || (area == TRACK_AREA_PAT && action != SLIDE_ACTION_OFFSET)) { - copy_v2_v2(data->smin, data->min); - copy_v2_v2(data->smax, data->max); + if ((area == TRACK_AREA_SEARCH) || + (area == TRACK_AREA_PAT && action != SLIDE_ACTION_OFFSET)) + { + if (data->corners) { + memcpy(data->scorners, data->corners, sizeof(data->scorners)); + } + else { + copy_v2_v2(data->smin, data->min); + copy_v2_v2(data->smax, data->max); + } } data->mval[0] = event->mval[0]; @@ -326,9 +341,7 @@ static SlideMarkerData *create_slide_marker_data(SpaceClip *sc, MovieTrackingTra return data; } -/* corner = 0: right-bottom corner, - * corner = 1: left-top corner */ -static int mouse_on_corner(SpaceClip *sc, MovieTrackingTrack *track, MovieTrackingMarker *marker, +static int mouse_on_corner(SpaceClip *sc, MovieTrackingMarker *marker, int area, float co[2], int corner, int width, int height) { int inside = 0; @@ -337,12 +350,11 @@ static int mouse_on_corner(SpaceClip *sc, MovieTrackingTrack *track, MovieTracki float crn[2], dx, dy, tdx, tdy; if (area == TRACK_AREA_SEARCH) { - copy_v2_v2(min, track->search_min); - copy_v2_v2(max, track->search_max); + copy_v2_v2(min, marker->search_min); + copy_v2_v2(max, marker->search_max); } else { - copy_v2_v2(min, track->pat_min); - copy_v2_v2(max, track->pat_max); + BKE_tracking_marker_pattern_minmax(marker, min, max); } dx = size / width / sc->zoom; @@ -370,22 +382,95 @@ static int mouse_on_corner(SpaceClip *sc, MovieTrackingTrack *track, MovieTracki return inside; } +static int get_mouse_pattern_corner(SpaceClip *sc, MovieTrackingMarker *marker, float co[2], int width, int height) +{ + int i, next; + float len = FLT_MAX, dx, dy; + + for (i = 0; i < 4; i++) { + float cur_len; + + next = (i + 1) % 4; + + cur_len = len_v2v2(marker->pattern_corners[i], marker->pattern_corners[next]); + + len = MIN2(cur_len, len); + } + + dx = 6.0f / width / sc->zoom; + dy = 6.0f / height / sc->zoom; + + dx = MIN2(dx * 2.0f / 3.0f, len / 6.0f); + dy = MIN2(dy * 2.0f / 3.0f, len * width / height / 6.0f); + + for (i = 0; i < 4; i++) { + float crn[2]; + int inside; + + add_v2_v2v2(crn, marker->pattern_corners[i], marker->pos); + + inside = IN_RANGE_INCL(co[0], crn[0] - dx, crn[0] + dx) && + IN_RANGE_INCL(co[1], crn[1] - dy, crn[1] + dy); + + if (inside) + return i; + } + + return -1; +} + static int mouse_on_offset(SpaceClip *sc, MovieTrackingTrack *track, MovieTrackingMarker *marker, - float co[2], int width, int height) + float co[2], int width, int height) { float pos[2], dx, dy; + float pat_min[2], pat_max[2]; + + BKE_tracking_marker_pattern_minmax(marker, pat_min, pat_max); add_v2_v2v2(pos, marker->pos, track->offset); dx = 12.0f / width / sc->zoom; dy = 12.0f / height / sc->zoom; - dx = MIN2(dx, (track->pat_max[0] - track->pat_min[0]) / 2.0f); - dy = MIN2(dy, (track->pat_max[1] - track->pat_min[1]) / 2.0f); + dx = MIN2(dx, (pat_max[0] - pat_min[0]) / 2.0f); + dy = MIN2(dy, (pat_max[1] - pat_min[1]) / 2.0f); return co[0] >= pos[0] - dx && co[0] <= pos[0] + dx && co[1] >= pos[1] - dy && co[1] <= pos[1] + dy; } +static int slide_check_corners(float (*corners)[2]) +{ + int i, next, prev; + float cross = 0.0f; + float p[2] = {0.0f, 0.0f}; + + if (!isect_point_quad_v2(p, corners[0], corners[1], corners[2], corners[3])) + return FALSE; + + for (i = 0; i < 4; i++) { + float v1[2], v2[2], cur_cross; + + next = (i + 1) % 4; + prev = (4 + i - 1) % 4; + + sub_v2_v2v2(v1, corners[i], corners[prev]); + sub_v2_v2v2(v2, corners[next], corners[i]); + + cur_cross = cross_v2v2(v1, v2); + + if (fabsf(cur_cross) > FLT_EPSILON) { + if (cross == 0.0f) { + cross = cur_cross; + } + else if (cross * cur_cross < 0.0f) { + return FALSE; + } + } + } + + return TRUE; +} + static void hide_cursor(bContext *C) { wmWindow *win = CTX_wm_window(C); @@ -424,28 +509,45 @@ static void *slide_marker_customdata(bContext *C, wmEvent *event) MovieTrackingMarker *marker = BKE_tracking_get_marker(track, framenr); if ((marker->flag & MARKER_DISABLED) == 0) { - if (!customdata) + if (!customdata) { if (mouse_on_offset(sc, track, marker, co, width, height)) - customdata = create_slide_marker_data(sc, track, marker, event, TRACK_AREA_POINT, + customdata = create_slide_marker_data(sc, track, marker, event, TRACK_AREA_POINT, 0, SLIDE_ACTION_POS, width, height); + } if (sc->flag & SC_SHOW_MARKER_SEARCH) { - if (mouse_on_corner(sc, track, marker, TRACK_AREA_SEARCH, co, 1, width, height)) - customdata = create_slide_marker_data(sc, track, marker, event, TRACK_AREA_SEARCH, + if (mouse_on_corner(sc, marker, TRACK_AREA_SEARCH, co, 1, width, height)) { + customdata = create_slide_marker_data(sc, track, marker, event, TRACK_AREA_SEARCH, 0, SLIDE_ACTION_OFFSET, width, height); - else if (mouse_on_corner(sc, track, marker, TRACK_AREA_SEARCH, co, 0, width, height)) - customdata = create_slide_marker_data(sc, track, marker, event, TRACK_AREA_SEARCH, + } + else if (mouse_on_corner(sc, marker, TRACK_AREA_SEARCH, co, 0, width, height)) { + customdata = create_slide_marker_data(sc, track, marker, event, TRACK_AREA_SEARCH, 0, SLIDE_ACTION_SIZE, width, height); + } } if (!customdata && (sc->flag & SC_SHOW_MARKER_PATTERN)) { - if (mouse_on_corner(sc, track, marker, TRACK_AREA_PAT, co, 1, width, height)) - customdata = create_slide_marker_data(sc, track, marker, event, TRACK_AREA_PAT, - SLIDE_ACTION_OFFSET, width, height); - - if (!customdata && mouse_on_corner(sc, track, marker, TRACK_AREA_PAT, co, 0, width, height)) - customdata = create_slide_marker_data(sc, track, marker, event, TRACK_AREA_PAT, - SLIDE_ACTION_SIZE, width, height); + /* XXX: need to be real check if affine tracking is enabled, but for now not + * sure how to do this, so assume affine tracker is always enabled */ + if (TRUE) { + int corner = get_mouse_pattern_corner(sc, marker, co, width, height); + + if (corner != -1) { + customdata = create_slide_marker_data(sc, track, marker, event, TRACK_AREA_PAT, corner, + SLIDE_ACTION_POS, width, height); + } + } + else { + if (mouse_on_corner(sc, marker, TRACK_AREA_PAT, co, 1, width, height)) { + customdata = create_slide_marker_data(sc, track, marker, event, TRACK_AREA_PAT, 0, + SLIDE_ACTION_OFFSET, width, height); + } + + if (!customdata && mouse_on_corner(sc, marker, TRACK_AREA_PAT, co, 0, width, height)) { + customdata = create_slide_marker_data(sc, track, marker, event, TRACK_AREA_PAT, 0, + SLIDE_ACTION_SIZE, width, height); + } + } } if (customdata) @@ -493,9 +595,16 @@ static void cancel_mouse_slide(SlideMarkerData *data) copy_v2_v2(data->pos, data->spos); } else { - if (data->action == SLIDE_ACTION_SIZE) { - copy_v2_v2(data->min, data->smin); - copy_v2_v2(data->max, data->smax); + if ((data->action == SLIDE_ACTION_SIZE) || + (data->action == SLIDE_ACTION_POS && data->area == TRACK_AREA_PAT)) + { + if (data->corners) { + memcpy(data->corners, data->scorners, sizeof(data->scorners)); + } + else { + copy_v2_v2(data->min, data->smin); + copy_v2_v2(data->max, data->smax); + } } else { int a; @@ -531,6 +640,10 @@ static int slide_marker_modal(bContext *C, wmOperator *op, wmEvent *event) if (ELEM(event->type, LEFTCTRLKEY, RIGHTCTRLKEY)) data->lock = event->val == KM_RELEASE; + if (data->action == SLIDE_ACTION_POS) + if (ELEM(event->type, LEFTCTRLKEY, RIGHTCTRLKEY)) + data->scale = event->val == KM_PRESS; + if (ELEM(event->type, LEFTSHIFTKEY, RIGHTSHIFTKEY)) data->accurate = event->val == KM_PRESS; @@ -560,8 +673,6 @@ static int slide_marker_modal(bContext *C, wmOperator *op, wmEvent *event) else { data->pos[0] = data->spos[0] + dx; data->pos[1] = data->spos[1] + dy; - - data->marker->flag &= ~MARKER_TRACKED; } WM_event_add_notifier(C, NC_OBJECT | ND_TRANSFORM, NULL); @@ -569,18 +680,33 @@ static int slide_marker_modal(bContext *C, wmOperator *op, wmEvent *event) } else { if (data->action == SLIDE_ACTION_SIZE) { - data->min[0] = data->smin[0] - dx; - data->max[0] = data->smax[0] + dx; + if (data->corners) { + data->corners[0][0] = data->scorners[0][0] - dx; + data->corners[0][1] = data->scorners[0][1] + dy; + + data->corners[1][0] = data->scorners[1][0] + dx; + data->corners[1][1] = data->scorners[1][1] + dy; - data->min[1] = data->smin[1] + dy; - data->max[1] = data->smax[1] - dy; + data->corners[2][0] = data->scorners[2][0] + dx; + data->corners[2][1] = data->scorners[2][1] - dy; + + data->corners[3][0] = data->scorners[3][0] - dx; + data->corners[3][1] = data->scorners[3][1] - dy; + } + else { + data->min[0] = data->smin[0] - dx; + data->max[0] = data->smax[0] + dx; + + data->min[1] = data->smin[1] + dy; + data->max[1] = data->smax[1] - dy; + } if (data->area == TRACK_AREA_SEARCH) - BKE_tracking_clamp_track(data->track, CLAMP_SEARCH_DIM); + BKE_tracking_clamp_marker(data->marker, CLAMP_SEARCH_DIM); else - BKE_tracking_clamp_track(data->track, CLAMP_PAT_DIM); + BKE_tracking_clamp_marker(data->marker, CLAMP_PAT_DIM); } - else { + else if (data->action == SLIDE_ACTION_OFFSET) { float d[2] = {dx, dy}; if (data->area == TRACK_AREA_SEARCH) { @@ -597,10 +723,43 @@ static int slide_marker_modal(bContext *C, wmOperator *op, wmEvent *event) } if (data->area == TRACK_AREA_SEARCH) - BKE_tracking_clamp_track(data->track, CLAMP_SEARCH_POS); + BKE_tracking_clamp_marker(data->marker, CLAMP_SEARCH_POS); + } + else if (data->action == SLIDE_ACTION_POS) { + if (data->scale) { + float scale = 1.0f + 10.0f * (dx - dy); + + if (scale > 0.0f) { + int a; + + for (a = 0; a < 4; a++) { + mul_v2_v2fl(data->corners[a], data->scorners[a], scale); + } + } + } + else { + float spos[2]; + + copy_v2_v2(spos, data->pos); + + /* corners might've been scaled before, restore their original position */ + memcpy(data->corners, data->scorners, sizeof(data->scorners)); + + data->pos[0] = data->spos[0] + dx; + data->pos[1] = data->spos[1] + dy; + + if (!slide_check_corners(data->corners)) { + copy_v2_v2(data->pos, spos); + } + } + + /* currently only patterns are allowed to have such combination of event and data */ + BKE_tracking_clamp_marker(data->marker, CLAMP_PAT_DIM); } } + data->marker->flag &= ~MARKER_TRACKED; + WM_event_add_notifier(C, NC_MOVIECLIP | NA_EDITED, NULL); break; @@ -673,31 +832,41 @@ static int mouse_on_rect(float co[2], float pos[2], float min[2], float max[2], mouse_on_side(co, pos[0] + max[0], pos[1] + min[1], pos[0] + max[0], pos[1] + max[1], epsx, epsy); } +static int mouse_on_crns(float co[2], float pos[2], float crns[4][2], float epsx, float epsy) +{ + float dist = dist_to_crns(co, pos, crns); + + return dist < MAX2(epsx, epsy); +} + static int track_mouse_area(SpaceClip *sc, float co[2], MovieTrackingTrack *track) { int framenr = ED_space_clip_clip_framenr(sc); MovieTrackingMarker *marker = BKE_tracking_get_marker(track, framenr); + float pat_min[2], pat_max[2]; float epsx, epsy; int width, height; ED_space_clip_size(sc, &width, &height); - epsx = MIN4(track->pat_min[0] - track->search_min[0], track->search_max[0] - track->pat_max[0], - fabsf(track->pat_min[0]), fabsf(track->pat_max[0])) / 2; - epsy = MIN4(track->pat_min[1] - track->search_min[1], track->search_max[1] - track->pat_max[1], - fabsf(track->pat_min[1]), fabsf(track->pat_max[1])) / 2; + BKE_tracking_marker_pattern_minmax(marker, pat_min, pat_max); + + epsx = MIN4(pat_min[0] - marker->search_min[0], marker->search_max[0] - pat_max[0], + fabsf(pat_min[0]), fabsf(pat_max[0])) / 2; + epsy = MIN4(pat_min[1] - marker->search_min[1], marker->search_max[1] - pat_max[1], + fabsf(pat_min[1]), fabsf(pat_max[1])) / 2; epsx = MAX2(epsx, 2.0f / width); epsy = MAX2(epsy, 2.0f / height); if (sc->flag & SC_SHOW_MARKER_SEARCH) { - if (mouse_on_rect(co, marker->pos, track->search_min, track->search_max, epsx, epsy)) + if (mouse_on_rect(co, marker->pos, marker->search_min, marker->search_max, epsx, epsy)) return TRACK_AREA_SEARCH; } if ((marker->flag & MARKER_DISABLED) == 0) { if (sc->flag & SC_SHOW_MARKER_PATTERN) - if (mouse_on_rect(co, marker->pos, track->pat_min, track->pat_max, epsx, epsy)) + if (mouse_on_crns(co, marker->pos, marker->pattern_corners, epsx, epsy)) return TRACK_AREA_PAT; epsx = 12.0f / width; @@ -728,6 +897,21 @@ static float dist_to_rect(float co[2], float pos[2], float min[2], float max[2]) return MIN4(d1, d2, d3, d4); } +static float dist_to_crns(float co[2], float pos[2], float crns[4][2]) +{ + float d1, d2, d3, d4; + float p[2] = {co[0] - pos[0], co[1] - pos[1]}; + float *v1 = crns[0], *v2 = crns[1], + *v3 = crns[2], *v4 = crns[3]; + + d1 = dist_to_line_segment_v2(p, v1, v2); + d2 = dist_to_line_segment_v2(p, v2, v3); + d3 = dist_to_line_segment_v2(p, v3, v4); + d4 = dist_to_line_segment_v2(p, v4, v1); + + return MIN4(d1, d2, d3, d4); +} + static MovieTrackingTrack *find_nearest_track(SpaceClip *sc, ListBase *tracksbase, float co[2]) { MovieTrackingTrack *track = NULL, *cur; @@ -743,15 +927,15 @@ static MovieTrackingTrack *find_nearest_track(SpaceClip *sc, ListBase *tracksbas /* distance to marker point */ d1 = sqrtf((co[0] - marker->pos[0] - cur->offset[0]) * (co[0] - marker->pos[0] - cur->offset[0]) + - (co[1] - marker->pos[1] - cur->offset[1]) * (co[1] - marker->pos[1] - cur->offset[1])); + (co[1] - marker->pos[1] - cur->offset[1]) * (co[1] - marker->pos[1] - cur->offset[1])); /* distance to pattern boundbox */ if (sc->flag & SC_SHOW_MARKER_PATTERN) - d2 = dist_to_rect(co, marker->pos, cur->pat_min, cur->pat_max); + d2 = dist_to_crns(co, marker->pos, marker->pattern_corners); /* distance to search boundbox */ if (sc->flag & SC_SHOW_MARKER_SEARCH && TRACK_VIEW_SELECTED(sc, cur)) - d3 = dist_to_rect(co, marker->pos, cur->search_min, cur->search_max); + d3 = dist_to_rect(co, marker->pos, marker->search_min, marker->search_max); /* choose minimal distance. useful for cases of overlapped markers. */ dist = MIN3(d1, d2, d3); @@ -861,7 +1045,8 @@ void CLIP_OT_select(wmOperatorType *ot) /* api callbacks */ ot->exec = select_exec; ot->invoke = select_invoke; - ot->poll = ED_space_clip_tracking_poll; + //ot->poll = ED_space_clip_tracking_poll; // so mask view can Ctrl+RMB markers + ot->poll = ED_space_clip_view_clip_poll; /* flags */ ot->flag = OPTYPE_UNDO; diff --git a/source/blender/editors/transform/transform.c b/source/blender/editors/transform/transform.c index ea86083fac4..d41294c7875 100644 --- a/source/blender/editors/transform/transform.c +++ b/source/blender/editors/transform/transform.c @@ -815,6 +815,14 @@ int transformEvent(TransInfo *t, wmEvent *event) initSnapping(t, NULL); // need to reinit after mode change t->redraw |= TREDRAW_HARD; } + else if (t->mode == TFM_RESIZE) { + if (t->options & CTX_MOVIECLIP) { + restoreTransObjects(t); + + t->flag ^= T_ALT_TRANSFORM; + t->redraw |= TREDRAW_HARD; + } + } break; case TFM_MODAL_SNAP_INV_ON: @@ -2730,6 +2738,9 @@ static void ElementResize(TransInfo *t, TransData *td, float mat[3][3]) { copy_v3_v3(center, td->center); } + else if (t->options & CTX_MOVIECLIP) { + copy_v3_v3(center, td->center); + } else { copy_v3_v3(center, t->center); } @@ -3103,6 +3114,10 @@ static void ElementRotation(TransInfo *t, TransData *td, float mat[3][3], short { center = td->center; } + + if (t->options & CTX_MOVIECLIP) { + center = td->center; + } } if (t->flag & T_POINTS) { diff --git a/source/blender/editors/transform/transform_conversions.c b/source/blender/editors/transform/transform_conversions.c index ce65127c896..007ec3c5250 100644 --- a/source/blender/editors/transform/transform_conversions.c +++ b/source/blender/editors/transform/transform_conversions.c @@ -5588,23 +5588,24 @@ typedef struct TransDataTracking { short coord; } TransDataTracking; -static void markerToTransDataInit(TransData *td, TransData2D *td2d, TransDataTracking *tdt, MovieTrackingTrack *track, - int area, float loc[2], float rel[2], const float off[2]) +static void markerToTransDataInit(TransData *td, TransData2D *td2d, TransDataTracking *tdt, + MovieTrackingTrack *track, MovieTrackingMarker *marker, + int area, float loc[2], float rel[2], const float off[2], float aspx, float aspy) { int anchor = area == TRACK_AREA_POINT && off; tdt->mode = transDataTracking_ModeTracks; if (anchor) { - td2d->loc[0] = rel[0]; /* hold original location */ - td2d->loc[1] = rel[1]; + td2d->loc[0] = rel[0] * aspx; /* hold original location */ + td2d->loc[1] = rel[1] * aspy; tdt->loc= loc; td2d->loc2d = loc; /* current location */ } else { - td2d->loc[0] = loc[0]; /* hold original location */ - td2d->loc[1] = loc[1]; + td2d->loc[0] = loc[0] * aspx; /* hold original location */ + td2d->loc[1] = loc[1] * aspy; td2d->loc2d = loc; /* current location */ } @@ -5618,8 +5619,8 @@ static void markerToTransDataInit(TransData *td, TransData2D *td2d, TransDataTra if (rel) { if (!anchor) { - td2d->loc[0] += rel[0]; - td2d->loc[1] += rel[1]; + td2d->loc[0] += rel[0] * aspx; + td2d->loc[1] += rel[1] * aspy; } copy_v2_v2(tdt->srelative, rel); @@ -5630,9 +5631,12 @@ static void markerToTransDataInit(TransData *td, TransData2D *td2d, TransDataTra td->flag = 0; td->loc = td2d->loc; - copy_v3_v3(td->center, td->loc); copy_v3_v3(td->iloc, td->loc); + //copy_v3_v3(td->center, td->loc); + td->center[0] = marker->pos[0] * aspx; + td->center[1] = marker->pos[1] * aspy; + memset(td->axismtx, 0, sizeof(td->axismtx)); td->axismtx[2][2] = 1.0f; @@ -5647,27 +5651,37 @@ static void markerToTransDataInit(TransData *td, TransData2D *td2d, TransDataTra } static void trackToTransData(SpaceClip *sc, TransData *td, TransData2D *td2d, - TransDataTracking *tdt, MovieTrackingTrack *track) + TransDataTracking *tdt, MovieTrackingTrack *track, float aspx, float aspy) { int framenr = ED_space_clip_clip_framenr(sc); MovieTrackingMarker *marker = BKE_tracking_ensure_marker(track, framenr); tdt->flag = marker->flag; - marker->flag &= ~(MARKER_DISABLED|MARKER_TRACKED); + marker->flag &= ~(MARKER_DISABLED | MARKER_TRACKED); - markerToTransDataInit(td++, td2d++, tdt++, track, TRACK_AREA_POINT, track->offset, marker->pos, track->offset); + markerToTransDataInit(td++, td2d++, tdt++, track, marker, TRACK_AREA_POINT, + track->offset, marker->pos, track->offset, aspx, aspy); - if (track->flag & SELECT) - markerToTransDataInit(td++, td2d++, tdt++, track, TRACK_AREA_POINT, marker->pos, NULL, NULL); + if (track->flag & SELECT) { + markerToTransDataInit(td++, td2d++, tdt++, track, marker, TRACK_AREA_POINT, + marker->pos, NULL, NULL, aspx, aspy); + } if (track->pat_flag & SELECT) { - markerToTransDataInit(td++, td2d++, tdt++, track, TRACK_AREA_PAT, track->pat_min, marker->pos, NULL); - markerToTransDataInit(td++, td2d++, tdt++, track, TRACK_AREA_PAT, track->pat_max, marker->pos, NULL); + int a; + + for (a = 0; a < 4; a++) { + markerToTransDataInit(td++, td2d++, tdt++, track, marker, TRACK_AREA_PAT, + marker->pattern_corners[a], marker->pos, NULL, aspx, aspy); + } } if (track->search_flag & SELECT) { - markerToTransDataInit(td++, td2d++, tdt++, track, TRACK_AREA_SEARCH, track->search_min, marker->pos, NULL); - markerToTransDataInit(td++, td2d++, tdt++, track, TRACK_AREA_SEARCH, track->search_max, marker->pos, NULL); + markerToTransDataInit(td++, td2d++, tdt++, track, marker, TRACK_AREA_SEARCH, + marker->search_min, marker->pos, NULL, aspx, aspy); + + markerToTransDataInit(td++, td2d++, tdt++, track, marker, TRACK_AREA_SEARCH, + marker->search_max, marker->pos, NULL, aspx, aspy); } } @@ -5694,6 +5708,7 @@ static void createTransTrackingTracksData(bContext *C, TransInfo *t) MovieTrackingMarker *marker; TransDataTracking *tdt; int framenr = ED_space_clip_clip_framenr(sc); + float aspx, aspy; /* count */ t->total = 0; @@ -5709,7 +5724,7 @@ static void createTransTrackingTracksData(bContext *C, TransInfo *t) t->total++; if (track->pat_flag & SELECT) - t->total+= 2; + t->total+= 4; if (track->search_flag & SELECT) t->total+= 2; @@ -5721,6 +5736,8 @@ static void createTransTrackingTracksData(bContext *C, TransInfo *t) if (t->total == 0) return; + ED_space_clip_aspect_dimension_aware(sc, &aspx, &aspy); + td = t->data = MEM_callocN(t->total*sizeof(TransData), "TransTracking TransData"); td2d = t->data2d = MEM_callocN(t->total*sizeof(TransData2D), "TransTracking TransData2D"); tdt = t->customData = MEM_callocN(t->total*sizeof(TransDataTracking), "TransTracking TransDataTracking"); @@ -5733,25 +5750,23 @@ static void createTransTrackingTracksData(bContext *C, TransInfo *t) if (TRACK_VIEW_SELECTED(sc, track) && (track->flag & TRACK_LOCKED) == 0) { marker = BKE_tracking_get_marker(track, framenr); - trackToTransData(sc, td, td2d, tdt, track); + trackToTransData(sc, td, td2d, tdt, track, aspx, aspy); /* offset */ td++; td2d++; tdt++; - if ((marker->flag & MARKER_DISABLED) == 0) { - if (track->flag & SELECT) { - td++; - td2d++; - tdt++; - } + if (track->flag & SELECT) { + td++; + td2d++; + tdt++; + } - if (track->pat_flag & SELECT) { - td += 2; - td2d += 2; - tdt +=2; - } + if (track->pat_flag & SELECT) { + td += 4; + td2d += 4; + tdt += 4; } if (track->search_flag & SELECT) { @@ -5903,9 +5918,6 @@ static void createTransTrackingData(bContext *C, TransInfo *t) if (!clip || width == 0 || height == 0) return; - if (!ELEM(t->mode, TFM_RESIZE, TFM_TRANSLATION)) - return; - if (ar->regiontype == RGN_TYPE_PREVIEW) { /* transformation was called from graph editor */ createTransTrackingCurvesData(C, t); @@ -5973,10 +5985,14 @@ static void cancelTransTracking(TransInfo *t) void flushTransTracking(TransInfo *t) { + SpaceClip *sc = t->sa->spacedata.first; TransData *td; TransData2D *td2d; TransDataTracking *tdt; int a; + float aspx, aspy; + + ED_space_clip_aspect_dimension_aware(sc, &aspx, &aspy); if (t->state == TRANS_CANCEL) cancelTransTracking(t); @@ -5984,31 +6000,46 @@ void flushTransTracking(TransInfo *t) /* flush to 2d vector from internally used 3d vector */ for (a=0, td= t->data, td2d= t->data2d, tdt= t->customData; a<t->total; a++, td2d++, td++, tdt++) { if (tdt->mode == transDataTracking_ModeTracks) { - if (t->flag & T_ALT_TRANSFORM) { - if (tdt->area == TRACK_AREA_POINT && tdt->relative) { - float d[2], d2[2]; + float loc2d[2]; - if (!tdt->smarkers) { - tdt->smarkers = MEM_callocN(sizeof(*tdt->smarkers)*tdt->markersnr, "flushTransTracking markers"); - for (a = 0; a < tdt->markersnr; a++) - copy_v2_v2(tdt->smarkers[a], tdt->markers[a].pos); - } + if (t->mode == TFM_ROTATION && tdt->area == TRACK_AREA_SEARCH) { + continue; + } + + loc2d[0] = td2d->loc[0] / aspx; + loc2d[1] = td2d->loc[1] / aspy; + + if (t->flag & T_ALT_TRANSFORM) { + if (t->mode == TFM_RESIZE) { + if (tdt->area != TRACK_AREA_PAT) + continue; + } + else if (t->mode == TFM_TRANSLATION) { + if (tdt->area == TRACK_AREA_POINT && tdt->relative) { + float d[2], d2[2]; + + if (!tdt->smarkers) { + tdt->smarkers = MEM_callocN(sizeof(*tdt->smarkers)*tdt->markersnr, "flushTransTracking markers"); + for (a = 0; a < tdt->markersnr; a++) + copy_v2_v2(tdt->smarkers[a], tdt->markers[a].pos); + } - sub_v2_v2v2(d, td2d->loc, tdt->soffset); - sub_v2_v2(d, tdt->srelative); + sub_v2_v2v2(d, loc2d, tdt->soffset); + sub_v2_v2(d, tdt->srelative); - sub_v2_v2v2(d2, td2d->loc, tdt->srelative); + sub_v2_v2v2(d2, loc2d, tdt->srelative); - for (a= 0; a<tdt->markersnr; a++) - add_v2_v2v2(tdt->markers[a].pos, tdt->smarkers[a], d2); + for (a= 0; a<tdt->markersnr; a++) + add_v2_v2v2(tdt->markers[a].pos, tdt->smarkers[a], d2); - negate_v2_v2(td2d->loc2d, d); + negate_v2_v2(td2d->loc2d, d); + } } } if (tdt->area!=TRACK_AREA_POINT || tdt->relative==0) { - td2d->loc2d[0] = td2d->loc[0]; - td2d->loc2d[1] = td2d->loc[1]; + td2d->loc2d[0] = loc2d[0]; + td2d->loc2d[1] = loc2d[1]; if (tdt->relative) sub_v2_v2(td2d->loc2d, tdt->relative); diff --git a/source/blender/editors/transform/transform_generics.c b/source/blender/editors/transform/transform_generics.c index e0920e323aa..1b8cc14ecac 100644 --- a/source/blender/editors/transform/transform_generics.c +++ b/source/blender/editors/transform/transform_generics.c @@ -643,23 +643,30 @@ static void recalcData_spaceclip(TransInfo *t) MovieClip *clip = ED_space_clip(sc); ListBase *tracksbase = BKE_tracking_get_tracks(&clip->tracking); MovieTrackingTrack *track; + int framenr = sc->user.framenr; flushTransTracking(t); track = tracksbase->first; while (track) { if (TRACK_VIEW_SELECTED(sc, track) && (track->flag & TRACK_LOCKED)==0) { + MovieTrackingMarker *marker = BKE_tracking_get_marker(track, framenr); + if (t->mode == TFM_TRANSLATION) { if (TRACK_AREA_SELECTED(track, TRACK_AREA_PAT)) - BKE_tracking_clamp_track(track, CLAMP_PAT_POS); + BKE_tracking_clamp_marker(marker, CLAMP_PAT_POS); if (TRACK_AREA_SELECTED(track, TRACK_AREA_SEARCH)) - BKE_tracking_clamp_track(track, CLAMP_SEARCH_POS); + BKE_tracking_clamp_marker(marker, CLAMP_SEARCH_POS); } else if (t->mode == TFM_RESIZE) { if (TRACK_AREA_SELECTED(track, TRACK_AREA_PAT)) - BKE_tracking_clamp_track(track, CLAMP_PAT_DIM); + BKE_tracking_clamp_marker(marker, CLAMP_PAT_DIM); if (TRACK_AREA_SELECTED(track, TRACK_AREA_SEARCH)) - BKE_tracking_clamp_track(track, CLAMP_SEARCH_DIM); + BKE_tracking_clamp_marker(marker, CLAMP_SEARCH_DIM); + } + else if (t->mode == TFM_ROTATION) { + if (TRACK_AREA_SELECTED(track, TRACK_AREA_PAT)) + BKE_tracking_clamp_marker(marker, CLAMP_PAT_POS); } } diff --git a/source/blender/editors/transform/transform_ops.c b/source/blender/editors/transform/transform_ops.c index e4c85309081..52b32ae66fc 100644 --- a/source/blender/editors/transform/transform_ops.c +++ b/source/blender/editors/transform/transform_ops.c @@ -1025,6 +1025,7 @@ void transform_keymap_for_space(wmKeyConfig *keyconf, wmKeyMap *keymap, int spac WM_keymap_add_item(keymap, OP_TRANSLATION, GKEY, KM_PRESS, 0, 0); WM_keymap_add_item(keymap, OP_TRANSLATION, EVT_TWEAK_S, KM_ANY, 0, 0); WM_keymap_add_item(keymap, OP_RESIZE, SKEY, KM_PRESS, 0, 0); + WM_keymap_add_item(keymap, OP_ROTATION, RKEY, KM_PRESS, 0, 0); break; default: break; diff --git a/source/blender/makesdna/DNA_movieclip_types.h b/source/blender/makesdna/DNA_movieclip_types.h index fd7046854ff..b9d63167700 100644 --- a/source/blender/makesdna/DNA_movieclip_types.h +++ b/source/blender/makesdna/DNA_movieclip_types.h @@ -92,6 +92,9 @@ typedef struct MovieClip { typedef struct MovieClipScopes { int ok; /* 1 means scopes are ok and recalculation is unneeded */ int track_preview_height; /* height of track preview widget */ + int frame_width, frame_height; /* width and height of frame for which scopes are calculated */ + struct MovieTrackingMarker undist_marker; /* undistorted position of marker used for pattern sampling */ + struct ImBuf *track_search; /* search area of a track */ struct ImBuf *track_preview; /* ImBuf displayed in track preview */ float track_pos[2]; /* sub-pizel position of marker in track ImBuf */ short track_disabled; /* active track is disabled, special notifier should be drawn */ diff --git a/source/blender/makesdna/DNA_tracking_types.h b/source/blender/makesdna/DNA_tracking_types.h index 823ecbbbba6..c5b0174a3c9 100644 --- a/source/blender/makesdna/DNA_tracking_types.h +++ b/source/blender/makesdna/DNA_tracking_types.h @@ -35,6 +35,7 @@ #ifndef __DNA_TRACKING_TYPES_H__ #define __DNA_TRACKING_TYPES_H__ +#include "DNA_defs.h" #include "DNA_listBase.h" /* match-moving data */ @@ -69,6 +70,27 @@ typedef struct MovieTrackingCamera { typedef struct MovieTrackingMarker { float pos[2]; /* 2d position of marker on frame (in unified 0..1 space) */ + + /* corners of pattern in the following order: + * + * Y + * ^ + * | (3) --- (2) + * | | | + * | | | + * | | | + * | (0) --- (1) + * +-------------> X + * + * the coordinates are stored relative to pos. + */ + float pattern_corners[4][2]; + + /* positions of left-bottom and right-top corners of search area (in unified 0..1 units, + * relative to marker->pos + */ + float search_min[2], search_max[2]; + int framenr; /* number of frame marker is associated with */ int flag; /* Marker's flag (alive, ...) */ } MovieTrackingMarker; @@ -79,8 +101,19 @@ typedef struct MovieTrackingTrack { char name[64]; /* MAX_NAME */ /* ** setings ** */ - float pat_min[2], pat_max[2]; /* positions of left-bottom and right-top corners of pattern (in unified 0..1 space) */ - float search_min[2], search_max[2]; /* positions of left-bottom and right-top corners of search area (in unified 0..1 space) */ + + /* positions of left-bottom and right-top corners of pattern (in unified 0..1 units, + * relative to marker->pos) + * moved to marker's corners since planar tracking implementation + */ + float pat_min[2] DNA_DEPRECATED, pat_max[2] DNA_DEPRECATED; + + /* positions of left-bottom and right-top corners of search area (in unified 0..1 units, + * relative to marker->pos + * moved to marker since affine tracking implementation + */ + float search_min[2] DNA_DEPRECATED, search_max[2] DNA_DEPRECATED; + float offset[2]; /* offset to "parenting" point */ /* ** track ** */ @@ -96,35 +129,32 @@ typedef struct MovieTrackingTrack { int flag, pat_flag, search_flag; /* flags (selection, ...) */ float color[3]; /* custom color for track */ - /* tracking algorithm to use; can be KLT or SAD */ + /* ** control how tracking happens */ short frames_limit; /* number of frames to be tarcked during single tracking session (if TRACKING_FRAMES_LIMIT is set) */ short margin; /* margin from frame boundaries */ short pattern_match; /* re-adjust every N frames */ - short tracker; /* tracking algorithm used for this track */ - - /* ** KLT tracker settings ** */ - short pyramid_levels, pad2; /* number of pyramid levels to use for KLT tracking */ - - /* ** SAD tracker settings ** */ + /* tracking parameters */ + short motion_model; /* model of the motion for this track */ + int algorithm_flag; /* flags for the tracking algorithm (use brute, use esm, use pyramid, etc */ float minimum_correlation; /* minimal correlation which is still treated as successful tracking */ - struct bGPdata *gpd; /* grease-pencil data */ + struct bGPdata *gpd; /* grease-pencil data */ } MovieTrackingTrack; typedef struct MovieTrackingSettings { int flag; /* ** default tracker settings */ - short default_tracker; /* tracking algorithm used by default */ - short default_pyramid_levels; /* number of pyramid levels to use for KLT tracking */ - float default_minimum_correlation; /* minimal correlation which is still treated as successful tracking */ - short default_pattern_size; /* size of pattern area for new tracks */ - short default_search_size; /* size of search area for new tracks */ - short default_frames_limit; /* number of frames to be tarcked during single tracking session (if TRACKING_FRAMES_LIMIT is set) */ - short default_margin; /* margin from frame boundaries */ - short default_pattern_match; /* re-adjust every N frames */ - short default_flag; /* default flags like color channels used by default */ + short default_motion_model; /* model of the motion for this track */ + short default_algorithm_flag; /* flags for the tracking algorithm (use brute, use esm, use pyramid, etc */ + float default_minimum_correlation; /* minimal correlation which is still treated as successful tracking */ + short default_pattern_size; /* size of pattern area for new tracks */ + short default_search_size; /* size of search area for new tracks */ + short default_frames_limit; /* number of frames to be tarcked during single tracking session (if TRACKING_FRAMES_LIMIT is set) */ + short default_margin; /* margin from frame boundaries */ + short default_pattern_match; /* re-adjust every N frames */ + short default_flag; /* default flags like color channels used by default */ short motion_flag; /* flags describes motion type */ @@ -258,10 +288,17 @@ enum { #define TRACK_PREVIEW_GRAYSCALE (1<<9) #define TRACK_DOPE_SEL (1<<10) -/* MovieTrackingTrack->tracker */ -#define TRACKER_KLT 0 -#define TRACKER_SAD 1 -#define TRACKER_HYBRID 2 +/* MovieTrackingTrack->motion_model */ +#define TRACK_MOTION_MODEL_TRANSLATION 0 +#define TRACK_MOTION_MODEL_TRANSLATION_ROTATION 1 +#define TRACK_MOTION_MODEL_TRANSLATION_SCALE 2 +#define TRACK_MOTION_MODEL_TRANSLATION_ROTATION_SCALE 3 +#define TRACK_MOTION_MODEL_AFFINE 4 +#define TRACK_MOTION_MODEL_HOMOGRAPHY 5 + +/* MovieTrackingTrack->algorithm_flag */ +#define TRACK_ALGORITHM_FLAG_USE_BRUTE 1 +#define TRACK_ALGORITHM_FLAG_USE_NORMALIZATION 2 /* MovieTrackingTrack->adjframes */ #define TRACK_MATCH_KEYFRAME 0 diff --git a/source/blender/makesrna/RNA_access.h b/source/blender/makesrna/RNA_access.h index 8fc39c95d81..bf02ae50a94 100644 --- a/source/blender/makesrna/RNA_access.h +++ b/source/blender/makesrna/RNA_access.h @@ -343,8 +343,10 @@ extern StructRNA RNA_MotionPathVert; extern StructRNA RNA_MouseSensor; extern StructRNA RNA_MovieSequence; extern StructRNA RNA_MovieClipSequence; +extern StructRNA RNA_MovieTracking; extern StructRNA RNA_MovieTrackingTrack; extern StructRNA RNA_MovieTrackingObject; +extern StructRNA RNA_MovieTrackingTrack; extern StructRNA RNA_MulticamSequence; extern StructRNA RNA_MultiresModifier; extern StructRNA RNA_MusgraveTexture; diff --git a/source/blender/makesrna/intern/rna_tracking.c b/source/blender/makesrna/intern/rna_tracking.c index 17940dfca92..aaa96fc4d95 100644 --- a/source/blender/makesrna/intern/rna_tracking.c +++ b/source/blender/makesrna/intern/rna_tracking.c @@ -59,20 +59,6 @@ static char *rna_tracking_path(PointerRNA *UNUSED(ptr)) return BLI_sprintfN("tracking"); } -static void rna_tracking_defaultSettings_levelsUpdate(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr) -{ - MovieClip *clip = (MovieClip*)ptr->id.data; - MovieTracking *tracking = &clip->tracking; - MovieTrackingSettings *settings = &tracking->settings; - - if (settings->default_tracker == TRACKER_KLT) { - int max_pyramid_level_factor = 1 << (settings->default_pyramid_levels - 1); - float search_ratio = 2.3f * max_pyramid_level_factor; - - settings->default_search_size = settings->default_pattern_size*search_ratio; - } -} - static void rna_tracking_defaultSettings_patternUpdate(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr) { MovieClip *clip = (MovieClip*)ptr->id.data; @@ -209,37 +195,6 @@ static void rna_trackingTrack_select_set(PointerRNA *ptr, int value) } } -static void rna_tracking_trackerPattern_update(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr) -{ - MovieTrackingTrack *track = (MovieTrackingTrack *)ptr->data; - - BKE_tracking_clamp_track(track, CLAMP_PAT_DIM); -} - -static void rna_tracking_trackerSearch_update(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr) -{ - MovieTrackingTrack *track = (MovieTrackingTrack *)ptr->data; - - BKE_tracking_clamp_track(track, CLAMP_SEARCH_DIM); -} - -static void rna_tracking_trackerAlgorithm_update(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr) -{ - MovieTrackingTrack *track = (MovieTrackingTrack *)ptr->data; - - if (track->tracker == TRACKER_KLT) - BKE_tracking_clamp_track(track, CLAMP_PYRAMID_LEVELS); - else - BKE_tracking_clamp_track(track, CLAMP_SEARCH_DIM); -} - -static void rna_tracking_trackerPyramid_update(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr) -{ - MovieTrackingTrack *track = (MovieTrackingTrack *)ptr->data; - - BKE_tracking_clamp_track(track, CLAMP_PYRAMID_LEVELS); -} - static char *rna_trackingCamera_path(PointerRNA *UNUSED(ptr)) { return BLI_sprintfN("tracking.camera"); @@ -413,6 +368,20 @@ static void rna_trackingMarker_frame_set(PointerRNA *ptr, int value) } } +static void rna_tracking_markerPattern_update(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr) +{ + MovieTrackingMarker *marker = (MovieTrackingMarker *)ptr->data; + + BKE_tracking_clamp_marker(marker, CLAMP_PAT_DIM); +} + +static void rna_tracking_markerSearch_update(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr) +{ + MovieTrackingMarker *marker = (MovieTrackingMarker *)ptr->data; + + BKE_tracking_clamp_marker(marker, CLAMP_SEARCH_DIM); +} + /* API */ static void add_tracks_to_base(MovieClip *clip, MovieTracking *tracking, ListBase *tracksbase, int frame, int number) @@ -498,11 +467,19 @@ void rna_trackingMarkers_delete_frame(MovieTrackingTrack *track, int framenr) #else -static EnumPropertyItem tracker_items[] = { - {TRACKER_KLT, "KLT", 0, "KLT", - "Kanade–Lucas–Tomasi tracker which works with most of video clips, a bit slower than SAD"}, - {TRACKER_SAD, "SAD", 0, "SAD", "Sum of Absolute Differences tracker which can be used when KLT tracker fails"}, - {TRACKER_HYBRID, "Hybrid", 0, "Hybrid", "A hybrid tracker that uses SAD for rough tracking, KLT for refinement."}, +static EnumPropertyItem tracker_motion_model[] = { + {TRACK_MOTION_MODEL_HOMOGRAPHY, "Perspective", 0, "Perspective", + "Search for markers that are perspectively deformed (homography) between frames."}, + {TRACK_MOTION_MODEL_AFFINE, "Affine", 0, "Affine", + "Search for markers that are affine-deformed (t, r, k, and skew) between frames."}, + {TRACK_MOTION_MODEL_TRANSLATION_ROTATION_SCALE, "LocRotScale", 0, "LocRotScale", + "Search for markers that are translated, rotated, and scaled between frames."}, + {TRACK_MOTION_MODEL_TRANSLATION_SCALE, "LocScale", 0, "LocScale", + "Search for markers that are translated and scaled between frames."}, + {TRACK_MOTION_MODEL_TRANSLATION_ROTATION, "LocRot", 0, "LocRot", + "Search for markers that are translated and rotated between frames."}, + {TRACK_MOTION_MODEL_TRANSLATION, "Loc", 0, "Loc", + "Search for markers that are translated between frames."}, {0, NULL, 0, NULL, NULL}}; static EnumPropertyItem pattern_match_items[] = { @@ -511,6 +488,7 @@ static EnumPropertyItem pattern_match_items[] = { {0, NULL, 0, NULL, NULL}}; static int rna_matrix_dimsize_4x4[] = {4, 4}; +static int rna_matrix_dimsize_4x2[] = {4, 2}; static void rna_def_trackingSettings(BlenderRNA *brna) { @@ -627,14 +605,14 @@ static void rna_def_trackingSettings(BlenderRNA *brna) RNA_def_property_boolean_sdna(prop, NULL, "motion_flag", TRACKING_MOTION_TRIPOD); RNA_def_property_ui_text(prop, "Tripod Motion", "Use special solver to track a stable camera position, such as a tripod"); - /* limit frames */ + /* default_limit_frames */ prop = RNA_def_property(srna, "default_frames_limit", PROP_INT, PROP_NONE); RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); RNA_def_property_int_sdna(prop, NULL, "default_frames_limit"); RNA_def_property_range(prop, 0, SHRT_MAX); RNA_def_property_ui_text(prop, "Frames Limit", "Every tracking cycle, this number of frames are tracked"); - /* pattern match */ + /* default_pattern_match */ prop = RNA_def_property(srna, "default_pattern_match", PROP_ENUM, PROP_NONE); RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); RNA_def_property_enum_sdna(prop, NULL, "default_pattern_match"); @@ -642,40 +620,42 @@ static void rna_def_trackingSettings(BlenderRNA *brna) RNA_def_property_ui_text(prop, "Pattern Match", "Track pattern from given frame when tracking marker to next frame"); - /* margin */ + /* default_margin */ prop = RNA_def_property(srna, "default_margin", PROP_INT, PROP_NONE); RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); RNA_def_property_int_sdna(prop, NULL, "default_margin"); RNA_def_property_range(prop, 0, 300); RNA_def_property_ui_text(prop, "Margin", "Default distance from image boudary at which marker stops tracking"); - /* tracking algorithm */ - prop = RNA_def_property(srna, "default_tracker", PROP_ENUM, PROP_NONE); + /* default_tracking_motion_model */ + prop = RNA_def_property(srna, "default_motion_model", PROP_ENUM, PROP_NONE); RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); - RNA_def_property_enum_items(prop, tracker_items); - RNA_def_property_update(prop, 0, "rna_tracking_defaultSettings_levelsUpdate"); - RNA_def_property_ui_text(prop, "Tracker", "Default tracking algorithm to use"); + RNA_def_property_enum_items(prop, tracker_motion_model); + RNA_def_property_ui_text(prop, "Motion model", "Default motion model to use for tracking"); - /* pyramid level for pyramid klt tracking */ - prop = RNA_def_property(srna, "default_pyramid_levels", PROP_INT, PROP_NONE); - RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); - RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); - RNA_def_property_int_sdna(prop, NULL, "default_pyramid_levels"); - RNA_def_property_range(prop, 1, 16); - RNA_def_property_update(prop, 0, "rna_tracking_defaultSettings_levelsUpdate"); - RNA_def_property_ui_text(prop, "Pyramid levels", "Default number of pyramid levels (increase on blurry footage)"); + /* use_brute */ + prop = RNA_def_property(srna, "default_use_brute", PROP_BOOLEAN, PROP_NONE); + RNA_def_property_boolean_sdna(prop, NULL, "default_algorithm_flag", TRACK_ALGORITHM_FLAG_USE_BRUTE); + RNA_def_property_ui_text(prop, "Prepass", "Use a brute-force translation-only initialization when tracking"); + RNA_def_property_update(prop, NC_MOVIECLIP|ND_DISPLAY, NULL); - /* minmal correlation - only used for SAD tracker */ + /* default use_normalization */ + prop = RNA_def_property(srna, "default_use_normalization", PROP_BOOLEAN, PROP_NONE); + RNA_def_property_boolean_sdna(prop, NULL, "default_algorithm_flag", TRACK_ALGORITHM_FLAG_USE_NORMALIZATION); + RNA_def_property_ui_text(prop, "Normalize", "Normalize light intensities while tracking. Slower"); + RNA_def_property_update(prop, NC_MOVIECLIP|ND_DISPLAY, NULL); + + /* default minmal correlation */ prop = RNA_def_property(srna, "default_correlation_min", PROP_FLOAT, PROP_NONE); RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); RNA_def_property_float_sdna(prop, NULL, "default_minimum_correlation"); - RNA_def_property_range(prop, -1.0f, 1.0f); - RNA_def_property_ui_range(prop, -1.0f, 1.0f, 0.1, 3); + RNA_def_property_range(prop, 0.0f, 1.0f); + RNA_def_property_ui_range(prop, 0.0f, 1.0f, 0.05, 3); RNA_def_property_ui_text(prop, "Correlation", - "Default minimal value of correlation between matched pattern and reference " - "which is still treated as successful tracking"); + "Default minimum value of correlation between matched pattern and reference " + "that is still treated as successful tracking"); /* default pattern size */ prop = RNA_def_property(srna, "default_pattern_size", PROP_INT, PROP_NONE); @@ -693,19 +673,19 @@ static void rna_def_trackingSettings(BlenderRNA *brna) RNA_def_property_update(prop, 0, "rna_tracking_defaultSettings_searchUpdate"); RNA_def_property_ui_text(prop, "Search Size", "Size of search area for newly created tracks"); - /* use_red_channel */ + /* default use_red_channel */ prop = RNA_def_property(srna, "use_default_red_channel", PROP_BOOLEAN, PROP_NONE); RNA_def_property_boolean_negative_sdna(prop, NULL, "default_flag", TRACK_DISABLE_RED); RNA_def_property_ui_text(prop, "Use Red Channel", "Use red channel from footage for tracking"); RNA_def_property_update(prop, NC_MOVIECLIP|ND_DISPLAY, NULL); - /* use_green_channel */ + /* default_use_green_channel */ prop = RNA_def_property(srna, "use_default_green_channel", PROP_BOOLEAN, PROP_NONE); RNA_def_property_boolean_negative_sdna(prop, NULL, "default_flag", TRACK_DISABLE_GREEN); RNA_def_property_ui_text(prop, "Use Green Channel", "Use green channel from footage for tracking"); RNA_def_property_update(prop, NC_MOVIECLIP|ND_DISPLAY, NULL); - /* use_blue_channel */ + /* default_use_blue_channel */ prop = RNA_def_property(srna, "use_default_blue_channel", PROP_BOOLEAN, PROP_NONE); RNA_def_property_boolean_negative_sdna(prop, NULL, "default_flag", TRACK_DISABLE_BLUE); RNA_def_property_ui_text(prop, "Use Blue Channel", "Use blue channel from footage for tracking"); @@ -839,6 +819,38 @@ static void rna_def_trackingMarker(BlenderRNA *brna) RNA_def_property_boolean_sdna(prop, NULL, "flag", MARKER_DISABLED); RNA_def_property_ui_text(prop, "Mode", "Is marker muted for current frame"); RNA_def_property_update(prop, NC_MOVIECLIP|NA_EDITED, NULL); + + /* pattern */ + prop = RNA_def_property(srna, "pattern_corners", PROP_FLOAT, PROP_MATRIX); + RNA_def_property_float_sdna(prop, NULL, "pattern_corners"); + RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); + RNA_def_property_multi_array(prop, 2, rna_matrix_dimsize_4x2); + RNA_def_property_ui_range(prop, -FLT_MAX, FLT_MAX, 1, RNA_TRANSLATION_PREC_DEFAULT); + RNA_def_property_ui_text(prop, "Pattern Corners", + "Array of coordinates which represents patter's corners in " + " normalized coordinates relative to marker position"); + RNA_def_property_update(prop, NC_MOVIECLIP|NA_EDITED, "rna_tracking_markerPattern_update"); + + /* search */ + prop = RNA_def_property(srna, "search_min", PROP_FLOAT, PROP_TRANSLATION); + RNA_def_property_array(prop, 2); + RNA_def_property_ui_range(prop, -FLT_MAX, FLT_MAX, 1, RNA_TRANSLATION_PREC_DEFAULT); + RNA_def_property_float_sdna(prop, NULL, "search_min"); + RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); + RNA_def_property_ui_text(prop, "Search Min", + "Left-bottom corner of search area in normalized coordinates relative " + "to marker position"); + RNA_def_property_update(prop, NC_MOVIECLIP|NA_EDITED, "rna_tracking_markerSearch_update"); + + prop = RNA_def_property(srna, "search_max", PROP_FLOAT, PROP_TRANSLATION); + RNA_def_property_array(prop, 2); + RNA_def_property_ui_range(prop, -FLT_MAX, FLT_MAX, 1, RNA_TRANSLATION_PREC_DEFAULT); + RNA_def_property_float_sdna(prop, NULL, "search_max"); + RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); + RNA_def_property_ui_text(prop, "Search Max", + "Right-bottom corner of search area in normalized coordinates relative " + "to marker position"); + RNA_def_property_update(prop, NC_MOVIECLIP|NA_EDITED, "rna_tracking_markerSearch_update"); } static void rna_def_trackingMarkers(BlenderRNA *brna, PropertyRNA *cprop) @@ -899,48 +911,6 @@ static void rna_def_trackingTrack(BlenderRNA *brna) RNA_def_property_update(prop, NC_MOVIECLIP|NA_EDITED, NULL); RNA_def_struct_name_property(srna, prop); - /* Pattern */ - prop = RNA_def_property(srna, "pattern_min", PROP_FLOAT, PROP_TRANSLATION); - RNA_def_property_array(prop, 2); - RNA_def_property_ui_range(prop, -FLT_MAX, FLT_MAX, 1, RNA_TRANSLATION_PREC_DEFAULT); - RNA_def_property_float_sdna(prop, NULL, "pat_min"); - RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); - RNA_def_property_ui_text(prop, "Pattern Min", - "Left-bottom corner of pattern area in normalized coordinates relative " - "to marker position"); - RNA_def_property_update(prop, NC_MOVIECLIP|NA_EDITED, "rna_tracking_trackerPattern_update"); - - prop = RNA_def_property(srna, "pattern_max", PROP_FLOAT, PROP_TRANSLATION); - RNA_def_property_array(prop, 2); - RNA_def_property_ui_range(prop, -FLT_MAX, FLT_MAX, 1, RNA_TRANSLATION_PREC_DEFAULT); - RNA_def_property_float_sdna(prop, NULL, "pat_max"); - RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); - RNA_def_property_ui_text(prop, "Pattern Max", - "Right-bottom corner of pattern area in normalized coordinates relative " - "to marker position"); - RNA_def_property_update(prop, NC_MOVIECLIP|NA_EDITED, "rna_tracking_trackerPattern_update"); - - /* Search */ - prop = RNA_def_property(srna, "search_min", PROP_FLOAT, PROP_TRANSLATION); - RNA_def_property_array(prop, 2); - RNA_def_property_ui_range(prop, -FLT_MAX, FLT_MAX, 1, RNA_TRANSLATION_PREC_DEFAULT); - RNA_def_property_float_sdna(prop, NULL, "search_min"); - RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); - RNA_def_property_ui_text(prop, "Search Min", - "Left-bottom corner of search area in normalized coordinates relative " - "to marker position"); - RNA_def_property_update(prop, NC_MOVIECLIP|NA_EDITED, "rna_tracking_trackerSearch_update"); - - prop = RNA_def_property(srna, "search_max", PROP_FLOAT, PROP_TRANSLATION); - RNA_def_property_array(prop, 2); - RNA_def_property_ui_range(prop, -FLT_MAX, FLT_MAX, 1, RNA_TRANSLATION_PREC_DEFAULT); - RNA_def_property_float_sdna(prop, NULL, "search_max"); - RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); - RNA_def_property_ui_text(prop, "Search Max", - "Right-bottom corner of search area in normalized coordinates relative " - "to marker position"); - RNA_def_property_update(prop, NC_MOVIECLIP|NA_EDITED, "rna_tracking_trackerSearch_update"); - /* limit frames */ prop = RNA_def_property(srna, "frames_limit", PROP_INT, PROP_NONE); RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); @@ -966,32 +936,36 @@ static void rna_def_trackingTrack(BlenderRNA *brna) RNA_def_property_range(prop, 0, 300); RNA_def_property_ui_text(prop, "Margin", "Distance from image boudary at which marker stops tracking"); - /* tracking algorithm */ - prop = RNA_def_property(srna, "tracker", PROP_ENUM, PROP_NONE); - RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); - RNA_def_property_enum_items(prop, tracker_items); - RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); - RNA_def_property_ui_text(prop, "Tracker", "Tracking algorithm to use"); - RNA_def_property_update(prop, NC_MOVIECLIP|NA_EDITED, "rna_tracking_trackerAlgorithm_update"); - - /* pyramid level for pyramid klt tracking */ - prop = RNA_def_property(srna, "pyramid_levels", PROP_INT, PROP_NONE); + /* tracking motion model */ + prop = RNA_def_property(srna, "motion_model", PROP_ENUM, PROP_NONE); RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); - RNA_def_property_int_sdna(prop, NULL, "pyramid_levels"); + RNA_def_property_enum_items(prop, tracker_motion_model); RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); - RNA_def_property_range(prop, 1, 16); - RNA_def_property_ui_text(prop, "Pyramid levels", "Number of pyramid levels (increase on blurry footage)"); - RNA_def_property_update(prop, NC_MOVIECLIP|NA_EDITED, "rna_tracking_trackerPyramid_update"); + RNA_def_property_ui_text(prop, "Motion model", "Default motion model to use for tracking"); - /* minmal correlation - only used for SAD tracker */ + /* minimum correlation */ prop = RNA_def_property(srna, "correlation_min", PROP_FLOAT, PROP_NONE); RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); RNA_def_property_float_sdna(prop, NULL, "minimum_correlation"); - RNA_def_property_range(prop, -1.0f, 1.0f); - RNA_def_property_ui_range(prop, -1.0f, 1.0f, 0.1, 3); + RNA_def_property_range(prop, 0.0f, 1.0f); + RNA_def_property_ui_range(prop, 0.0f, 1.0f, 0.05, 3); RNA_def_property_ui_text(prop, "Correlation", "Minimal value of correlation between matched pattern and reference " - "which is still treated as successful tracking"); + "that is still treated as successful tracking"); + + /* use_brute */ + prop = RNA_def_property(srna, "use_brute", PROP_BOOLEAN, PROP_NONE); + RNA_def_property_boolean_sdna(prop, NULL, "algorithm_flag", TRACK_ALGORITHM_FLAG_USE_BRUTE); + RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); + RNA_def_property_ui_text(prop, "Prepass", "Use a brute-force translation only pre-track before refinement"); + RNA_def_property_update(prop, NC_MOVIECLIP|ND_DISPLAY, NULL); + + /* use_brute */ + prop = RNA_def_property(srna, "use_normalization", PROP_BOOLEAN, PROP_NONE); + RNA_def_property_boolean_sdna(prop, NULL, "algorithm_flag", TRACK_ALGORITHM_FLAG_USE_NORMALIZATION); + RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); + RNA_def_property_ui_text(prop, "Normalize", "Normalize light intensities while tracking. Slower"); + RNA_def_property_update(prop, NC_MOVIECLIP|ND_DISPLAY, NULL); /* markers */ prop = RNA_def_property(srna, "markers", PROP_COLLECTION, PROP_NONE); |