diff options
43 files changed, 4254 insertions, 890 deletions
diff --git a/extern/libmv/CMakeLists.txt b/extern/libmv/CMakeLists.txt index 09c36fc4383..69632b31685 100644 --- a/extern/libmv/CMakeLists.txt +++ b/extern/libmv/CMakeLists.txt @@ -71,14 +71,20 @@ if(WITH_LIBMV) ) list(APPEND SRC + intern/autotrack.cc intern/camera_intrinsics.cc intern/detector.cc + intern/frame_accessor.cc intern/homography.cc intern/image.cc intern/logging.cc intern/reconstruction.cc intern/track_region.cc intern/tracks.cc + intern/tracksN.cc + libmv/autotrack/autotrack.cc + libmv/autotrack/predict_tracks.cc + libmv/autotrack/tracks.cc libmv/base/aligned_malloc.cc libmv/image/array_nd.cc libmv/image/convolve.cc @@ -114,14 +120,27 @@ if(WITH_LIBMV) libmv/tracking/trklt_region_tracker.cc + intern/autotrack.h intern/camera_intrinsics.h intern/detector.h + intern/frame_accessor.h intern/homography.h intern/image.h intern/logging.h intern/reconstruction.h intern/track_region.h intern/tracks.h + intern/tracksN.h + libmv/autotrack/autotrack.h + libmv/autotrack/callbacks.h + libmv/autotrack/frame_accessor.h + libmv/autotrack/marker.h + libmv/autotrack/model.h + libmv/autotrack/predict_tracks.h + libmv/autotrack/quad.h + libmv/autotrack/reconstruction.h + libmv/autotrack/region.h + libmv/autotrack/tracks.h libmv/base/aligned_malloc.h libmv/base/id_generator.h libmv/base/scoped_ptr.h @@ -171,6 +190,7 @@ if(WITH_LIBMV) libmv/simple_pipeline/tracks.h libmv/tracking/brute_region_tracker.h libmv/tracking/hybrid_region_tracker.h + libmv/tracking/kalman_filter.h libmv/tracking/klt_region_tracker.h libmv/tracking/pyramid_region_tracker.h libmv/tracking/region_tracker.h @@ -197,6 +217,8 @@ if(WITH_LIBMV) if(WITH_GTESTS) blender_add_lib(libmv_test_dataset "./libmv/multiview/test_data_sets.cc" "" "") + BLENDER_SRC_GTEST("libmv_predict_tracks" "./libmv/autotrack/predict_tracks_test.cc" "libmv_test_dataset;extern_libmv;extern_ceres") + BLENDER_SRC_GTEST("libmv_tracks" "./libmv/autotrack/tracks_test.cc" "libmv_test_dataset;extern_libmv;extern_ceres") BLENDER_SRC_GTEST("libmv_scoped_ptr" "./libmv/base/scoped_ptr_test.cc" "libmv_test_dataset;extern_libmv;extern_ceres") BLENDER_SRC_GTEST("libmv_vector" "./libmv/base/vector_test.cc" "libmv_test_dataset;extern_libmv;extern_ceres") BLENDER_SRC_GTEST("libmv_array_nd" "./libmv/image/array_nd_test.cc" "libmv_test_dataset;extern_libmv;extern_ceres") @@ -240,7 +262,7 @@ if(WITH_LIBMV) endif() # make GLog a separate target, so it can be used for gtest as well. -if(WITH_LIBMV OR WITH_GTESTS OR WITH_CYCLES_LOGGING) +if(WITH_LIBMV OR WITH_GTESTS) # We compile GLog together with GFlag so we don't worry about # adding extra lib to linker. set(GLOG_SRC diff --git a/extern/libmv/ChangeLog b/extern/libmv/ChangeLog index 276fbb2ff0d..5c2129b6b1a 100644 --- a/extern/libmv/ChangeLog +++ b/extern/libmv/ChangeLog @@ -1,3 +1,365 @@ +commit d976e034cdf74b34860e0632d7b29713f47c5756 +Author: Keir Mierle <mierle@gmail.com> +Date: Sat Aug 23 00:38:01 2014 -0700 + + Minor keyframe selection cleanups + + Reviewers: sergey + + Reviewed By: sergey + + Differential Revision: https://developer.blender.org/D757 + +commit bc99ca55dadfca89fde0f93764397c2fe028943d +Author: Sergey Sharybin <sergey.vfx@gmail.com> +Date: Sat Aug 23 01:55:32 2014 +0600 + + implement backward prediction + + The title actually says it all, just extend current implementation + of PredictMarkerPosition() to cases when tracking happens in the reverse + order (from the end frame to start). + + it's still doesn't solve all the ambiguity happening in the function + in cases when one tracks the feature and then re-tracks it in order + to refine the sliding. This is considered a separate TODO for now and + will likely be solved by passing tracking direction to the prediction + function. + + Reviewers: keir + + Reviewed By: keir + + Differential Revision: https://developer.blender.org/D663 + +commit 5b87682d98df65ade02638bc6482d824cf0dd0b3 +Author: Keir Mierle <mierle@gmail.com> +Date: Thu Aug 21 22:45:22 2014 -0700 + + Make libmv compile on Ubuntu 14.04 + + Reviewers: fsiddi + + Reviewed By: fsiddi + + Subscribers: sergey + + Differential Revision: https://developer.blender.org/D755 + +commit 0a81db623c458e0384b4f7060d1bcff8993fb469 +Author: Sergey Sharybin <sergey.vfx@gmail.com> +Date: Wed Jul 23 00:42:00 2014 +0600 + + Fix wrong residual blocks counter + + This happened in cases when having zero-weighted tracks + and could lead to some assert failures on marking parameter + block constant. + +commit 2824dbac54cacf74828678be7a5c9fd960ce83e2 +Author: Sergey Sharybin <sergey.vfx@gmail.com> +Date: Fri Jul 18 12:52:03 2014 +0600 + + Fix search area sliding issue + + The only way to do this is to store search region in floats + and round when we need to sample it. Otherwise you'll always + have sliding effect caused by rounding the issues, especially + when doing incremental offset (thing which happens in the + prediction code). + + Pretty much straightforward change apart from stuff to be kept + in mind: offset calculation int should happen relative to the + rounded search region. This is because tracker works in the space + of the search window image which get's rounded on the frame access, + + This makes API a bit creepy because frame accessor uses the same + Region struct as the search window in Marker and ideally we would + need to have either IntRegion or Region<int> in order to make + Libmv fully track on what's getting rounded and when. + + Reviewers: keir + + Reviewed By: keir + + Differential Revision: https://developer.blender.org/D616 + +commit 04862c479332308be47a0f27361402444ace8880 +Author: Keir Mierle <mierle@gmail.com> +Date: Fri May 9 23:00:03 2014 +0200 + + Start the automatic 2D tracking code + + This starts the 2D automatic tracking code. It is totally unfinished. + + Reviewers: sergey + + Reviewed By: sergey + + Differential Revision: https://developer.blender.org/D523 + +commit be679f67d807a2139c1f7d7e2ca45141940b30d5 +Author: Keir Mierle <mierle@gmail.com> +Date: Fri May 9 14:36:04 2014 +0200 + + Also shift the search window + + Reviewers: sergey + + Reviewed By: sergey + + Differential Revision: https://developer.blender.org/D520 + +commit 66b8f5eef2633ebcde32a388fc14c60171011821 +Author: Keir Mierle <mierle@gmail.com> +Date: Fri May 9 13:06:28 2014 +0200 + + Change the search region to absolute frame coordinates + + Smarter Eigen usage + + Better error logging + + Reviewers: sergey + + Reviewed By: sergey + + Differential Revision: https://developer.blender.org/D519 + +commit a08193319ae409fad8f08887eae1f79f02e91eaa +Author: Keir Mierle <mierle@gmail.com> +Date: Fri May 9 12:02:47 2014 +0200 + + First cut at predictive tracing + + This adds a Kalman filter-based approach to predict where a marker + will go in the next frame to track. Hopefully this will make the + tracker work faster by avoiding lengthy searches. This code + compiles, but is otherwise untested, and likely does not work. + + Fix else branch + + Add some tests + + Update patch coordinates as well (and test) + + Reviewers: sergey + + Reviewed By: sergey + + Differential Revision: https://developer.blender.org/D518 + +commit 607ffb2f62b56e34a841abbb952d83e19cd1e23c +Author: Keir Mierle <mierle@gmail.com> +Date: Thu May 8 16:05:28 2014 +0200 + + Add constructor to AutoTrack + +commit c39e20a0c27da3733804c3848454b5d4c4f0e66b +Author: Keir Mierle <mierle@gmail.com> +Date: Thu May 8 16:04:20 2014 +0200 + + Fix GetMarker compilation issue + +commit 8dd93e431b6e44439c803bfd26ec2669b656177e +Author: Keir Mierle <mierle@gmail.com> +Date: Thu May 8 15:50:26 2014 +0200 + + Expose GetMarker() in AutoTrack + + Reviewers: sergey + + Reviewed By: sergey + + Differential Revision: https://developer.blender.org/D516 + +commit 4405dff60ea08d454b64da1a7c0595d9328cf8a3 +Author: Keir Mierle <mierle@gmail.com> +Date: Thu May 8 15:38:14 2014 +0200 + + Add public SetMarkers to AutoTrack + + Reviewers: sergey + + Reviewed By: sergey + + Differential Revision: https://developer.blender.org/D515 + +commit c90837f6db276a3b1f610eaad509155f6a43b24f +Author: Keir Mierle <mierle@gmail.com> +Date: Thu May 8 15:17:48 2014 +0200 + + Make autotrack skeleton compile + + Reviewers: sergey + + Reviewed By: sergey + + Differential Revision: https://developer.blender.org/D514 + +commit be01baa2e82e36f63e548f073157e68d2ff870c0 +Author: Keir Mierle <mierle@gmail.com> +Date: Wed May 7 18:48:55 2014 +0200 + + Add preliminary TrackMarkerToFrame in autotrack + + Reviewers: sergey + + Reviewed By: sergey + + Differential Revision: https://developer.blender.org/D509 + +commit 0cab028d591b3d08672ca86eb6c6e4ac1aacf1d0 +Author: Sergey Sharybin <sergey.vfx@gmail.com> +Date: Wed May 7 17:59:11 2014 +0200 + + Remove assert from ArrayND Resize + + That assert broke initialization of arrays which doesn't + own the data since constructor uses Resize to set shape + and strides. + + Strides are still to be fixed, but that's for later. + +commit 64f9c118029a9351e9023e96527c120e1d724d5b +Author: Sergey Sharybin <sergey.vfx@gmail.com> +Date: Wed May 7 17:42:21 2014 +0200 + + Fix ArrayND freeing the data it doesn't own + + Can't really guarantee it works fully correct now, + but at least this check is needed anyway and compilation + works just fine. + + Reviewers: keir + + Reviewed By: keir + + Differential Revision: https://developer.blender.org/D508 + +commit 0618f1c8e88dfc738cdde55784da80b889905e7c +Author: Keir Mierle <mierle@gmail.com> +Date: Wed May 7 12:03:32 2014 +0200 + + Minor changes + + Reviewers: sergey + + Reviewed By: sergey + + Differential Revision: https://developer.blender.org/D505 + +commit 5c34335e1bb90c4ed701ee830c718ed4e20dbffa +Author: Sergey Sharybin <sergey.vfx@gmail.com> +Date: Wed May 7 11:12:23 2014 +0200 + + Fix compilation error in frame accessor + + - int64 is not a standard type, we've got int64_t defined in + std int. We also have an msvc port of this header, so should + not be an issue. + + - Fixed inconsistency in usage of CacheKey and Key, used Key. + + - Some functions weren't marked as virtual. + + Additional change: added self to authors. + + Reviewers: keir + + Reviewed By: keir + + Differential Revision: https://developer.blender.org/D504 + +commit 06bc207614e262cd688e2c3ed820ade7c77bdb66 +Author: Keir Mierle <mierle@gmail.com> +Date: Tue May 6 22:30:59 2014 +0200 + + Start new Tracks implementation + + This adds the new Tracks implementation, as well as a + trivial test to show it compiles. + + Reviewers: sergey + + Reviewed By: sergey + + Differential Revision: https://developer.blender.org/D502 + +commit 25ce061e6da69881460ba7718bb0d660a2380a02 +Author: Keir Mierle <mierle@gmail.com> +Date: Tue May 6 19:10:51 2014 +0200 + + Add Reconstruction class for new API + + This starts the new Reconstruction class (with support for e.g. planes). This + also starts the new namespace "mv" which will eventually have all the symbols + we wish to export. + + Reviewers: sergey + + Reviewed By: sergey + + Differential Revision: https://developer.blender.org/D501 + +commit 0a6af3e29016048978aea607673340500e050339 +Author: Keir Mierle <mierle@gmail.com> +Date: Tue May 6 17:52:53 2014 +0200 + + Add a new Tracks implementation + + Reviewers: sergey + + Reviewed By: sergey + + Differential Revision: https://developer.blender.org/D500 + +commit 887b68d29c2b198f4939f9ab5153881aa2c1806e +Author: Keir Mierle <mierle@gmail.com> +Date: Tue May 6 17:01:39 2014 +0200 + + Initial commit of unfinished AutoTrack API + + This starts the creating the new AutoTrack API. The new API will + make it possible for libmv to do full autotracking, including + predictive tracking and also support multiple motion models (3D + planes etc). + + The first goal (not in this patch) is to convert Blender to use + the new API without adding any new functionality. + + Note: This does not add any of the API to the build system! + It likely does not compile. + + Reviewers: sergey + + Reviewed By: sergey + + Differential Revision: https://developer.blender.org/D499 + +commit 08cc227d431d257d27f300fbb8e6991e663302da +Author: Sergey Sharybin <sergey.vfx@gmail.com> +Date: Tue May 6 13:09:22 2014 +0200 + + Fix homography test failure + + It was caused by assuming that reconstructed homography matrix + should look exactly the same as the matrix used to generate a + test case. + + It's not actually valid assumption because different-looking + matrices could correspond to the same exact transform. + + In this change we make it so actual "re-projected" vectors + are being checked, not the values in matrix. This makes it + more predictable verification. + + Reviewers: keir + + Reviewed By: keir + + Differential Revision: https://developer.blender.org/D488 + commit 0b7d83dc9627447dc7df64d7e3a468aefe9ddc13 Author: Sergey Sharybin <sergey.vfx@gmail.com> Date: Wed Apr 23 19:14:55 2014 +0600 @@ -338,342 +700,3 @@ Date: Thu Feb 13 23:55:03 2014 +0600 Actually we're to switch to external Ceres rather than bundled one, would make life much easier actually. - -commit b1381540305d69c702eb2f051bd543fb5c1c3e2c -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Thu Feb 6 18:01:58 2014 +0600 - - Made FAST detector optional - - This way it's possible to bundle Libmv sources subset - to applications which doesn't need FAST detector. - - Mainly this is done for Blender integration. - -commit da4607f010bca0b3532cd4444afbb10bc774fc32 -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Tue Jan 28 18:32:39 2014 +0600 - - Implemented scoped_array and use it in detector - - scoped_array is pretty much the same as scoped_ptr - with the only difference that it'll free memory using - delete[] operator. - - It also gives some additional API functions to access - array elements. - - Currently it only used to manage images denoted as byte - arrays in detector. - - Reviewers: keir - - Reviewed By: keir - - Differential Revision: https://developer.blender.org/D266 - -commit cd7eb3eff2e69ce5e08570ead83ae6d35ee48857 -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Tue Jan 28 17:23:47 2014 +0600 - - Improvements to weighted tracks behavior - - First thing changed by this commit is making it so - Euclidean intersection takes track weight into account - when solving minimization problem. This behaves the - same exact way as it is for BA step. - - Second thing is related on how average reprojection error - is being calculated. It didn't take track weight into - account which could confuse users. Now average reprojection - error will give the same result as intersection/BA uses - during minimization which gives much more predictable - behavior. - - Reviewers: keir - - Reviewed By: keir - - CC: sebastian_k - - Differential Revision: https://developer.blender.org/D265 - -commit 6559b36dc14369175bfa0830323146acd3426483 -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Tue Jan 28 16:39:14 2014 +0600 - - Fixes for keyframe selection - - Using tracks with constant zero weight used to crash - keyframe selection since it was trying to use missing - parameter blocks for Jacobian evaluation, - - Also fixed possible issues with wrong camera block being - marked as variable. This could technically happen when - having zero weighted tracks. Made it so all camera blocks - are marked as variable for now. - -commit 557d531b061aa69d114e89cbb325c5175389afec -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Tue Jan 28 16:10:33 2014 +0600 - - Style cleanup: wrong indentation of wrapped line - -commit ca15262cf07a873268173965ee1fb84f9729b744 -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Tue Jan 28 15:21:36 2014 +0600 - - Rework detector API and implement Harris detector - - Switch the detector API to a single function which accepts - a float image and detector options. This makes usage of - feature detection more unified across different algorithms. - - Options structure is pretty much straightforward and contains - detector to be used and all the detector-specific settings. - - Also implemented Harris feature detection algorithm which - is not as fast as FAST one but is expected to detect more - robust feature points. - - Reviewers: keir - - Reviewed By: keir - - Differential Revision: https://developer.blender.org/D258 - -commit 6458915f64fceba108c5279b7320ca8c76e8a742 -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Fri Jan 24 19:14:18 2014 +0600 - - Add arcanist configuration file - -commit 0a69fadadc5aacbd339f839ac5bd12c3571278b1 -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Thu Jan 9 15:50:11 2014 +0600 - - Fix compilation error on NetBSD - - - NetBSD doesn't provide sincos(3) in libm, so don't try to use it - - Use posix_memalign on NetBSD - - Original patch is by Jeorg Sonnenberger to Blender patch tracker, thanks! - -commit b0df3e291e6c85f791658be04334efafc41989f5 -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Thu Jan 2 15:12:18 2014 +0600 - - Fix build configuration warnings - - Those warnings were mainly caused by installation - configuration of Ceres. Made some tweaks to make - CMake happy for now. - - But for sure bigger cleanup here is needed. - -commit b68de6acd20f3ffab92e0cd450198a700cd109ab -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Thu Jan 2 15:04:05 2014 +0600 - - Code and style cleanup - - Mainly fixed some style warnings reported by cpplint. - - Also changed how camera (un)distortion happens internally - by replacing number of channels as a template argument - with number as channels passing as function argument. - Makes code easier to follow by eliminating loads checks - how much channels are used and which argument to pass to - the template. - -commit b9e467e7c077b58199c4110f6967b7c18d1e7bf7 -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Tue Dec 31 20:34:39 2013 +0600 - - Update Ceres to the latest upstream - - This brings up much easier termination type usage, - which for us means we might use: - - ceres::Summary::IsSolutionUsable() - - instead of doing manual funky enum values check. - -commit 3aeb2367e50b52ca2b9d59d4f0f0b4bbfd6a05e8 -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Tue Dec 31 20:43:24 2013 +0600 - - Update gtest to latest version 1.7.0 - - Also reshuffled CMakeLists in order to avoid - conflicts happening between gflags bundled to - Ceres and third_party. - -commit 30aaa9cd0b4a4eb0948a17824e7e7622d8ebcb41 -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Tue Dec 31 20:10:06 2013 +0600 - - Changes for VC2013 - - Solves compilation error when using msvc2013 - - Original patch is by Alexandr Kuznetsov to blender.git - -commit b204c0d25065a2b149de256929ff37d8f00f45bb -Author: Keir Mierle <mierle@gmail.com> -Date: Tue Dec 31 20:05:58 2013 +0600 - - Eagerly attempt to refine a track before doing a brute search - - Before the refinement phase of tracking, a brute force SAD search - is run across the search area. This works well but is slow; - especially if the guess for the track's location is accurate. - - This patch runs a refinement phase before running a brute force - search, hoping that the guessed position (in x2, y2) is close to - the best answer. If it is, then no brute search is done. If it is - not, then a normal brute force search followed by refinement is - done. - - In some cases this may produce worse tracks than before; the - regressions will need investigation. The predictive motion model - (to be implemented) will reduce the probability of that happening. - -commit 5361513f0328ff94b53125d29129561bb03132e8 -Author: Keir Mierle <mierle@gmail.com> -Date: Tue Dec 31 20:04:46 2013 +0600 - - Fix bug where libmv tracking incorrectly succeeds on failure - - Before this patch, if Ceres returned USER_SUCCESS indicating that - Ceres was only changing the tracked quad slightly between - iterations (indicating convergence), no final correlation check - was done. This leads to incorrectly returning that the tracking - was successful, when it actually failed. - -commit ba9e63eed09e33a48bbcb081058f45ac16f8738e -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Tue Dec 31 20:00:46 2013 +0600 - - Implementation of weighted tracks - - Added a weight field to Track structure which means - how much affect this track will have on the final - reconstruction. - - Currently it affects on BA step only which in most - cases will work just fine. However, it worth looking - into weight support for intersection/resection. - -commit 4600df8b685ca8c4daa22d6c3b0125fd42c3bc67 -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Tue Dec 31 19:30:14 2013 +0600 - - Code cleanup: move function prototype to header file - -commit 0ce5b6efde774b3f042acf9e42c95674548f1c01 -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Tue Dec 31 19:26:48 2013 +0600 - - Get rid of Allow Fallback option for euclidean resection - - It was rather confusing from the user usage point of view - and didn't get so much improvement after new bundle adjuster - was added. - - In the future we might want to switch resection to PPnP algorithm, - which could also might be a nice alternative to fallback option. - -commit 5d063426f4809000c27f38ed798e4224bbd09a6d -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Tue Dec 31 19:24:05 2013 +0600 - - Use explicit declaration of int types sign - - Mainly needs to make blender happy with this custom - header which used to run into conflict with other int - types headers. - - Wouldn't harm being more explicit here anyway. - -commit c5be59dee94f94de369006c544080282cfb245cc -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Tue Dec 31 14:50:00 2013 +0600 - - Implement refinement of estimated homography/fundamental matrices - - It was only possible to perform algebraic estimation, which didn't - give so much accurate results. - - Implemented a way to perform algebraic estimation followed with - refinement step using Ceres minimizer. - - The code was actually mostly already there since keyframe selection - patch. Made such estimation a generic function in multiview/ and - hanged API for estimation in order to pass all additional options via - an options structure (the same way as it's done fr Ceres). - - Most of the options are straight-forward to understand,but some - deserves more description here: - - * expected_average_symmetric_distance is used as an early output check - and as soon as average symmetric error goes below this threshold - refining finishes. - - This distance is measured in the same units as input points are. - - It is arguable whether we need callback for this or not, but seems - Ceres doesn't have some kind of absolute threshold for function value - and function_tolerance behaves different from logic behind expected - symmetric error. - - * There' an option to normalize correspondences before estimating the - homography in order to increase estimation stability. See - - R. Hartley and A. Zisserman. Multiple View Geometry in Computer - Vision. Cambridge University Press, second edition, 2003. - - https://www.cs.ubc.ca/grads/resources/thesis/May09/Dubrofsky_Elan.pdf - -commit 1cdad972c4a9005e78891524cbd6d65600ca7e99 -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Wed Sep 25 16:12:29 2013 +0600 - - Code cleanup: Minor function capitalization fix - - Original patch by Joseph Mansfield to the Blender repository. - -commit 434316d084e8a41fd452f03610d7244d664948dc -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Wed Sep 25 16:07:43 2013 +0600 - - Code cleanup: spelling correction - - Original patch by Joseph Mansfield to the Blender repository. - -commit 5cfe8465ac70407c0959c53bcd5206657a1322a2 -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Wed Sep 25 16:02:48 2013 +0600 - - Fix for uninitialized covariance matrix - - Lead to unpredictable tracking termination criteria. - -commit fd86b77d413489649a989f075b061714ed9a72fc -Author: Sergey Sharybin <sergey.vfx@gmail.com> -Date: Wed Sep 25 16:01:19 2013 +0600 - - Add Procrustes PNP ("PPnP") resection algorithm to libmv - - This adds a new Euclidean resection method, used to create the - initial reconstruction in the motion tracker, to libmv. The method - is based on the Procrustes PNP algorithm (aka "PPnP"). Currently - the algorithm is not connected with the motion tracker, but it - will be eventually since it supports initialization. - - Having an initial guess when doing resection is important for - ambiguous cases where potentially the user could offer extra - guidance to the solver, in the form of "this point is in front of - that point". - - Original patch by Keir Mierle made to Blender repository. diff --git a/extern/libmv/SConscript b/extern/libmv/SConscript index 162fb88df53..96282f3c3c2 100644 --- a/extern/libmv/SConscript +++ b/extern/libmv/SConscript @@ -6,7 +6,6 @@ import sys import os -from FindSharedPtr import FindSharedPtr Import('env') @@ -29,6 +28,7 @@ if env['WITH_BF_LIBMV']: src = env.Glob('intern/*.cc') src.remove('intern' + os.sep + 'stub.cc') + src += env.Glob('libmv/autotrack/*.cc') src += env.Glob('libmv/base/*.cc') src += env.Glob('libmv/image/*.cc') src += env.Glob('libmv/multiview/*.cc') diff --git a/extern/libmv/bundle.sh b/extern/libmv/bundle.sh index f352f77373b..1ac377e17fe 100755 --- a/extern/libmv/bundle.sh +++ b/extern/libmv/bundle.sh @@ -159,30 +159,36 @@ if(WITH_LIBMV) list(APPEND INC_SYS ../Eigen3 - \${PNG_INCLUDE_DIR} + \${PNG_INCLUDE_DIRS} \${ZLIB_INCLUDE_DIRS} ) list(APPEND SRC + intern/autotrack.cc intern/camera_intrinsics.cc intern/detector.cc + intern/frame_accessor.cc intern/homography.cc intern/image.cc intern/logging.cc intern/reconstruction.cc intern/track_region.cc intern/tracks.cc + intern/tracksN.cc ${sources} ${third_sources} + intern/autotrack.h intern/camera_intrinsics.h intern/detector.h + intern/frame_accessor.h intern/homography.h intern/image.h intern/logging.h intern/reconstruction.h intern/track_region.h intern/tracks.h + intern/tracksN.h ${headers} ${third_headers} diff --git a/extern/libmv/files.txt b/extern/libmv/files.txt index f1eb9580ff5..a206ffe1edd 100644 --- a/extern/libmv/files.txt +++ b/extern/libmv/files.txt @@ -1,3 +1,18 @@ +libmv/autotrack/autotrack.cc +libmv/autotrack/autotrack.h +libmv/autotrack/callbacks.h +libmv/autotrack/frame_accessor.h +libmv/autotrack/marker.h +libmv/autotrack/model.h +libmv/autotrack/predict_tracks.cc +libmv/autotrack/predict_tracks.h +libmv/autotrack/predict_tracks_test.cc +libmv/autotrack/quad.h +libmv/autotrack/reconstruction.h +libmv/autotrack/region.h +libmv/autotrack/tracks.cc +libmv/autotrack/tracks.h +libmv/autotrack/tracks_test.cc libmv/base/aligned_malloc.cc libmv/base/aligned_malloc.h libmv/base/id_generator.h @@ -104,6 +119,7 @@ libmv/tracking/brute_region_tracker.h libmv/tracking/brute_region_tracker_test.cc libmv/tracking/hybrid_region_tracker.cc libmv/tracking/hybrid_region_tracker.h +libmv/tracking/kalman_filter.h libmv/tracking/klt_region_tracker.cc libmv/tracking/klt_region_tracker.h libmv/tracking/klt_region_tracker_test.cc diff --git a/extern/libmv/intern/autotrack.cc b/extern/libmv/intern/autotrack.cc new file mode 100644 index 00000000000..f0cbc68f11e --- /dev/null +++ b/extern/libmv/intern/autotrack.cc @@ -0,0 +1,99 @@ +/* + * ***** BEGIN GPL LICENSE BLOCK ***** + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2014 Blender Foundation. + * All rights reserved. + * + * Contributor(s): Blender Foundation, + * Sergey Sharybin + * + * ***** END GPL LICENSE BLOCK ***** + */ + +#include "intern/autotrack.h" +#include "intern/tracksN.h" +#include "intern/utildefines.h" +#include "libmv/autotrack/autotrack.h" + +using mv::AutoTrack; +using mv::FrameAccessor; +using mv::Marker; +using libmv::TrackRegionOptions; +using libmv::TrackRegionResult; + +libmv_AutoTrack* libmv_autoTrackNew(libmv_FrameAccessor *frame_accessor) { + return (libmv_AutoTrack*) LIBMV_OBJECT_NEW(AutoTrack, + (FrameAccessor*) frame_accessor); +} + +void libmv_autoTrackDestroy(libmv_AutoTrack* libmv_autotrack) { + LIBMV_OBJECT_DELETE(libmv_autotrack, AutoTrack); +} + +void libmv_autoTrackSetOptions(libmv_AutoTrack* libmv_autotrack, + const libmv_AutoTrackOptions* options) { + AutoTrack *autotrack = ((AutoTrack*) libmv_autotrack); + libmv_configureTrackRegionOptions(options->track_region, + &autotrack->options.track_region); + + autotrack->options.search_region.min(0) = options->search_region.min[0]; + autotrack->options.search_region.min(1) = options->search_region.min[1]; + autotrack->options.search_region.max(0) = options->search_region.max[0]; + autotrack->options.search_region.max(1) = options->search_region.max[1]; +} + +int libmv_autoTrackMarker(libmv_AutoTrack* libmv_autotrack, + const libmv_TrackRegionOptions* libmv_options, + libmv_Marker *libmv_tracked_marker, + libmv_TrackRegionResult* libmv_result) { + + Marker tracked_marker; + TrackRegionOptions options; + TrackRegionResult result; + libmv_apiMarkerToMarker(*libmv_tracked_marker, &tracked_marker); + libmv_configureTrackRegionOptions(*libmv_options, + &options); + (((AutoTrack*) libmv_autotrack)->TrackMarker(&tracked_marker, + &result, + &options)); + libmv_markerToApiMarker(tracked_marker, libmv_tracked_marker); + libmv_regionTrackergetResult(result, libmv_result); + return result.is_usable(); +} + +void libmv_autoTrackAddMarker(libmv_AutoTrack* libmv_autotrack, + const libmv_Marker* libmv_marker) { + Marker marker; + libmv_apiMarkerToMarker(*libmv_marker, &marker); + ((AutoTrack*) libmv_autotrack)->AddMarker(marker); +} + +int libmv_autoTrackGetMarker(libmv_AutoTrack* libmv_autotrack, + int clip, + int frame, + int track, + libmv_Marker *libmv_marker) { + Marker marker; + int ok = ((AutoTrack*) libmv_autotrack)->GetMarker(clip, + frame, + track, + &marker); + if (ok) { + libmv_markerToApiMarker(marker, libmv_marker); + } + return ok; +} diff --git a/extern/libmv/intern/autotrack.h b/extern/libmv/intern/autotrack.h new file mode 100644 index 00000000000..22e530f6b2b --- /dev/null +++ b/extern/libmv/intern/autotrack.h @@ -0,0 +1,71 @@ +/* + * ***** BEGIN GPL LICENSE BLOCK ***** + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2014 Blender Foundation. + * All rights reserved. + * + * Contributor(s): Blender Foundation, + * Sergey Sharybin + * + * ***** END GPL LICENSE BLOCK ***** + */ + +#ifndef LIBMV_C_API_AUTOTRACK_H_ +#define LIBMV_C_API_AUTOTRACK_H_ + +#include "intern/track_region.h" +#include "intern/region.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct libmv_FrameAccessor libmv_FrameAccessor; +typedef struct libmv_AutoTrack libmv_AutoTrack; +typedef struct libmv_Marker libmv_Marker; + +typedef struct libmv_AutoTrackOptions { + libmv_TrackRegionOptions track_region; + libmv_Region search_region; +} libmv_AutoTrackOptions; + +libmv_AutoTrack* libmv_autoTrackNew(libmv_FrameAccessor *frame_accessor); + +void libmv_autoTrackDestroy(libmv_AutoTrack* libmv_autotrack); + +void libmv_autoTrackSetOptions(libmv_AutoTrack* libmv_autotrack, + const libmv_AutoTrackOptions* options); + +int libmv_autoTrackMarker(libmv_AutoTrack* libmv_autotrack, + const libmv_TrackRegionOptions* libmv_options, + libmv_Marker *libmv_tracker_marker, + libmv_TrackRegionResult* libmv_result); + +void libmv_autoTrackAddMarker(libmv_AutoTrack* libmv_autotrack, + const libmv_Marker* libmv_marker); + +int libmv_autoTrackGetMarker(libmv_AutoTrack* libmv_autotrack, + int clip, + int frame, + int track, + libmv_Marker *libmv_marker); + +#ifdef __cplusplus +} +#endif + +#endif // LIBMV_C_API_TRACKS_H_ diff --git a/extern/libmv/intern/frame_accessor.cc b/extern/libmv/intern/frame_accessor.cc new file mode 100644 index 00000000000..a7d969af05b --- /dev/null +++ b/extern/libmv/intern/frame_accessor.cc @@ -0,0 +1,164 @@ +/* + * ***** BEGIN GPL LICENSE BLOCK ***** + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2014 Blender Foundation. + * All rights reserved. + * + * Contributor(s): Blender Foundation, + * Sergey Sharybin + * + * ***** END GPL LICENSE BLOCK ***** + */ + +#include "intern/frame_accessor.h" +#include "intern/image.h" +#include "intern/utildefines.h" +#include "libmv/autotrack/frame_accessor.h" +#include "libmv/autotrack/region.h" +#include "libmv/image/image.h" + +namespace { + +using libmv::FloatImage; +using mv::FrameAccessor; +using mv::Region; + +struct LibmvFrameAccessor : public FrameAccessor { + LibmvFrameAccessor(libmv_FrameAccessorUserData* user_data, + libmv_GetImageCallback get_image_callback, + libmv_ReleaseImageCallback release_image_callback) + : user_data_(user_data), + get_image_callback_(get_image_callback), + release_image_callback_(release_image_callback) { } + + libmv_InputMode get_libmv_input_mode(InputMode input_mode) { + switch (input_mode) { +#define CHECK_INPUT_MODE(mode) \ + case mode: \ + return LIBMV_IMAGE_MODE_ ## mode; + CHECK_INPUT_MODE(MONO) + CHECK_INPUT_MODE(RGBA) +#undef CHECK_INPUT_MODE + } + assert(!"unknown input mode passed from Libmv."); + // TODO(sergey): Proper error handling here in the future. + return LIBMV_IMAGE_MODE_MONO; + } + + void get_libmv_region(const Region& region, + libmv_Region* libmv_region) { + libmv_region->min[0] = region.min(0); + libmv_region->min[1] = region.min(1); + libmv_region->max[0] = region.max(0); + libmv_region->max[1] = region.max(1); + } + + Key GetImage(int clip, + int frame, + InputMode input_mode, + int downscale, + const Region* region, + const Transform* transform, + FloatImage* destination) { + float *float_buffer; + int width, height, channels; + libmv_Region libmv_region; + if (region) { + get_libmv_region(*region, &libmv_region); + } + Key cache_key = get_image_callback_(user_data_, + clip, + frame, + get_libmv_input_mode(input_mode), + downscale, + region != NULL ? &libmv_region : NULL, + (libmv_FrameTransform*) transform, + &float_buffer, + &width, + &height, + &channels); + + // TODO(sergey): Dumb code for until we can set data directly. + FloatImage temp_image(float_buffer, + height, + width, + channels); + destination->CopyFrom(temp_image); + + return cache_key; + } + + void ReleaseImage(Key cache_key) { + release_image_callback_(cache_key); + } + + bool GetClipDimensions(int clip, int *width, int *height) { + return false; + } + + int NumClips() { + return 1; + } + + int NumFrames(int clip) { + return 0; + } + + libmv_FrameAccessorUserData* user_data_; + libmv_GetImageCallback get_image_callback_; + libmv_ReleaseImageCallback release_image_callback_; +}; + +} // namespace + +libmv_FrameAccessor* libmv_FrameAccessorNew( + libmv_FrameAccessorUserData* user_data, + libmv_GetImageCallback get_image_callback, + libmv_ReleaseImageCallback release_image_callback) { + return (libmv_FrameAccessor*) LIBMV_OBJECT_NEW(LibmvFrameAccessor, + user_data, + get_image_callback, + release_image_callback); +} + +void libmv_FrameAccessorDestroy(libmv_FrameAccessor* frame_accessor) { + LIBMV_OBJECT_DELETE(frame_accessor, LibmvFrameAccessor); +} + +int64_t libmv_frameAccessorgetTransformKey(const libmv_FrameTransform *transform) { + return ((FrameAccessor::Transform*) transform)->key(); +} + +void libmv_frameAccessorgetTransformRun(const libmv_FrameTransform *transform, + const libmv_FloatImage *input_image, + libmv_FloatImage *output_image) { + const FloatImage input(input_image->buffer, + input_image->width, + input_image->height, + input_image->channels); + + FloatImage output; + ((FrameAccessor::Transform*) transform)->run(input, + &output); + + int num_pixels = output.Width() *output.Height() * output.Depth(); + output_image->buffer = new float[num_pixels]; + memcpy(output_image->buffer, output.Data(), num_pixels * sizeof(float)); + output_image->width = output.Width(); + output_image->height = output.Height(); + output_image->channels = output.Depth(); +} diff --git a/extern/libmv/intern/frame_accessor.h b/extern/libmv/intern/frame_accessor.h new file mode 100644 index 00000000000..3e77b57c2e9 --- /dev/null +++ b/extern/libmv/intern/frame_accessor.h @@ -0,0 +1,79 @@ +/* + * ***** BEGIN GPL LICENSE BLOCK ***** + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2014 Blender Foundation. + * All rights reserved. + * + * Contributor(s): Blender Foundation, + * Sergey Sharybin + * + * ***** END GPL LICENSE BLOCK ***** + */ + +#ifndef LIBMV_C_API_FRAME_ACCESSOR_H_ +#define LIBMV_C_API_FRAME_ACCESSOR_H_ + +#include <stdint.h> + +#include "intern/region.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct libmv_FloatImage libmv_FloatImage; +typedef struct libmv_FrameAccessor libmv_FrameAccessor; +typedef struct libmv_FrameTransform libmv_FrameTransform; +typedef struct libmv_FrameAccessorUserData libmv_FrameAccessorUserData; +typedef void *libmv_CacheKey; + +typedef enum { + LIBMV_IMAGE_MODE_MONO, + LIBMV_IMAGE_MODE_RGBA, +} libmv_InputMode; + +typedef libmv_CacheKey (*libmv_GetImageCallback) ( + libmv_FrameAccessorUserData* user_data, + int clip, + int frame, + libmv_InputMode input_mode, + int downscale, + const libmv_Region* region, + const libmv_FrameTransform* transform, + float** destination, + int* width, + int* height, + int* channels); + +typedef void (*libmv_ReleaseImageCallback) (libmv_CacheKey cache_key); + +libmv_FrameAccessor* libmv_FrameAccessorNew( + libmv_FrameAccessorUserData* user_data, + libmv_GetImageCallback get_image_callback, + libmv_ReleaseImageCallback release_image_callback); +void libmv_FrameAccessorDestroy(libmv_FrameAccessor* frame_accessor); + +int64_t libmv_frameAccessorgetTransformKey(const libmv_FrameTransform *transform); + +void libmv_frameAccessorgetTransformRun(const libmv_FrameTransform *transform, + const libmv_FloatImage *input_image, + libmv_FloatImage *output_image); +#ifdef __cplusplus +} +#endif + +#endif // LIBMV_C_API_FRAME_ACCESSOR_H_ diff --git a/extern/libmv/intern/image.cc b/extern/libmv/intern/image.cc index df9a01ee8de..5018caef5b1 100644 --- a/extern/libmv/intern/image.cc +++ b/extern/libmv/intern/image.cc @@ -34,7 +34,7 @@ using libmv::FloatImage; using libmv::SamplePlanarPatch; -void libmv_floatImaheDestroy(libmv_FloatImage *image) { +void libmv_floatImageDestroy(libmv_FloatImage *image) { delete [] image->buffer; } diff --git a/extern/libmv/intern/image.h b/extern/libmv/intern/image.h index 9936e748b9d..1213943aac4 100644 --- a/extern/libmv/intern/image.h +++ b/extern/libmv/intern/image.h @@ -64,7 +64,7 @@ typedef struct libmv_FloatImage { int channels; } libmv_FloatImage; -void libmv_floatImaheDestroy(libmv_FloatImage *image); +void libmv_floatImageDestroy(libmv_FloatImage *image); void libmv_samplePlanarPatchFloat(const float* image, int width, diff --git a/extern/libmv/intern/stub.cc b/extern/libmv/intern/stub.cc index cd8bb8ab841..f307d831475 100644 --- a/extern/libmv/intern/stub.cc +++ b/extern/libmv/intern/stub.cc @@ -97,6 +97,10 @@ void libmv_samplePlanarPatchByte(const unsigned char * /*image*/, /* TODO(sergey): implement */ } +void libmv_floatImageDestroy(libmv_FloatImage* /*image*/) +{ +} + /* ************ Tracks ************ */ libmv_Tracks *libmv_tracksNew(void) { @@ -328,3 +332,68 @@ void libmv_homography2DFromCorrespondencesEuc(/* const */ double (*x1)[2], H[1][1] = 1.0f; H[2][2] = 1.0f; } + +/* ************ autotrack ************ */ + +libmv_AutoTrack* libmv_autoTrackNew(libmv_FrameAccessor* /*frame_accessor*/) +{ + return NULL; +} + +void libmv_autoTrackDestroy(libmv_AutoTrack* /*libmv_autotrack*/) +{ +} + +void libmv_autoTrackSetOptions(libmv_AutoTrack* /*libmv_autotrack*/, + const libmv_AutoTrackOptions* /*options*/) +{ +} + +int libmv_autoTrackMarker(libmv_AutoTrack* /*libmv_autotrack*/, + const libmv_TrackRegionOptions* /*libmv_options*/, + libmv_Marker */*libmv_tracker_marker*/, + libmv_TrackRegionResult* /*libmv_result*/) +{ + return 0; +} + +void libmv_autoTrackAddMarker(libmv_AutoTrack* /*libmv_autotrack*/, + const libmv_Marker* /*libmv_marker*/) +{ +} + +int libmv_autoTrackGetMarker(libmv_AutoTrack* /*libmv_autotrack*/, + int /*clip*/, + int /*frame*/, + int /*track*/, + libmv_Marker* /*libmv_marker*/) +{ + return 0; +} + +/* ************ frame accessor ************ */ + +libmv_FrameAccessor* libmv_FrameAccessorNew( + libmv_FrameAccessorUserData* /*user_data**/, + libmv_GetImageCallback /*get_image_callback*/, + libmv_ReleaseImageCallback /*release_image_callback*/) +{ + return NULL; +} + +void libmv_FrameAccessorDestroy(libmv_FrameAccessor* /*frame_accessor*/) +{ +} + +int64_t libmv_frameAccessorgetTransformKey( + const libmv_FrameTransform */*transform*/) +{ + return 0; +} + +void libmv_frameAccessorgetTransformRun(const libmv_FrameTransform* /*transform*/, + const libmv_FloatImage* /*input_image*/, + libmv_FloatImage* /*output_image*/) +{ +} + diff --git a/extern/libmv/intern/tracksN.cc b/extern/libmv/intern/tracksN.cc new file mode 100644 index 00000000000..9d68bce2869 --- /dev/null +++ b/extern/libmv/intern/tracksN.cc @@ -0,0 +1,136 @@ +/* + * ***** BEGIN GPL LICENSE BLOCK ***** + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2011 Blender Foundation. + * All rights reserved. + * + * Contributor(s): Blender Foundation, + * Sergey Sharybin + * + * ***** END GPL LICENSE BLOCK ***** + */ + +#include "intern/tracksN.h" +#include "intern/utildefines.h" +#include "libmv/autotrack/marker.h" +#include "libmv/autotrack/tracks.h" + +using mv::Marker; +using mv::Tracks; + +void libmv_apiMarkerToMarker(const libmv_Marker& libmv_marker, + Marker *marker) { + marker->clip = libmv_marker.clip; + marker->frame = libmv_marker.frame; + marker->track = libmv_marker.track; + marker->center(0) = libmv_marker.center[0]; + marker->center(1) = libmv_marker.center[1]; + for (int i = 0; i < 4; i++) { + marker->patch.coordinates(i, 0) = libmv_marker.patch[i][0]; + marker->patch.coordinates(i, 1) = libmv_marker.patch[i][1]; + } + marker->search_region.min(0) = libmv_marker.search_region_min[0]; + marker->search_region.min(1) = libmv_marker.search_region_min[1]; + marker->search_region.max(0) = libmv_marker.search_region_max[0]; + marker->search_region.max(1) = libmv_marker.search_region_max[1]; + marker->weight = libmv_marker.weight; + marker->source = (Marker::Source) libmv_marker.source; + marker->status = (Marker::Status) libmv_marker.status; + marker->reference_clip = libmv_marker.reference_clip; + marker->reference_frame = libmv_marker.reference_frame; + marker->model_type = (Marker::ModelType) libmv_marker.model_type; + marker->model_id = libmv_marker.model_id; +} + +void libmv_markerToApiMarker(const Marker& marker, + libmv_Marker *libmv_marker) { + libmv_marker->clip = marker.clip; + libmv_marker->frame = marker.frame; + libmv_marker->track = marker.track; + libmv_marker->center[0] = marker.center(0); + libmv_marker->center[1] = marker.center(1); + for (int i = 0; i < 4; i++) { + libmv_marker->patch[i][0] = marker.patch.coordinates(i, 0); + libmv_marker->patch[i][1] = marker.patch.coordinates(i, 1); + } + libmv_marker->search_region_min[0] = marker.search_region.min(0); + libmv_marker->search_region_min[1] = marker.search_region.min(1); + libmv_marker->search_region_max[0] = marker.search_region.max(0); + libmv_marker->search_region_max[1] = marker.search_region.max(1); + libmv_marker->weight = marker.weight; + libmv_marker->source = (libmv_MarkerSource) marker.source; + libmv_marker->status = (libmv_MarkerStatus) marker.status; + libmv_marker->reference_clip = marker.reference_clip; + libmv_marker->reference_frame = marker.reference_frame; + libmv_marker->model_type = (libmv_MarkerModelType) marker.model_type; + libmv_marker->model_id = marker.model_id; +} + +libmv_TracksN* libmv_tracksNewN(void) { + Tracks* tracks = LIBMV_OBJECT_NEW(Tracks); + + return (libmv_TracksN*) tracks; +} + +void libmv_tracksDestroyN(libmv_TracksN* libmv_tracks) { + LIBMV_OBJECT_DELETE(libmv_tracks, Tracks); +} + +void libmv_tracksAddMarkerN(libmv_TracksN* libmv_tracks, + const libmv_Marker* libmv_marker) { + Marker marker; + libmv_apiMarkerToMarker(*libmv_marker, &marker); + ((Tracks*) libmv_tracks)->AddMarker(marker); +} + +void libmv_tracksGetMarkerN(libmv_TracksN* libmv_tracks, + int clip, + int frame, + int track, + libmv_Marker* libmv_marker) { + Marker marker; + ((Tracks*) libmv_tracks)->GetMarker(clip, frame, track, &marker); + libmv_markerToApiMarker(marker, libmv_marker); +} + +void libmv_tracksRemoveMarkerN(libmv_TracksN* libmv_tracks, + int clip, + int frame, + int track) { + ((Tracks *) libmv_tracks)->RemoveMarker(clip, frame, track); +} + +void libmv_tracksRemoveMarkersForTrack(libmv_TracksN* libmv_tracks, + int track) { + ((Tracks *) libmv_tracks)->RemoveMarkersForTrack(track); +} + +int libmv_tracksMaxClipN(libmv_TracksN* libmv_tracks) { + return ((Tracks*) libmv_tracks)->MaxClip(); +} + +int libmv_tracksMaxFrameN(libmv_TracksN* libmv_tracks, int clip) { + return ((Tracks*) libmv_tracks)->MaxFrame(clip); +} + +int libmv_tracksMaxTrackN(libmv_TracksN* libmv_tracks) { + return ((Tracks*) libmv_tracks)->MaxTrack(); +} + +int libmv_tracksNumMarkersN(libmv_TracksN* libmv_tracks) { + return ((Tracks*) libmv_tracks)->NumMarkers(); +} diff --git a/extern/libmv/intern/tracksN.h b/extern/libmv/intern/tracksN.h new file mode 100644 index 00000000000..ea8b3e8a4f8 --- /dev/null +++ b/extern/libmv/intern/tracksN.h @@ -0,0 +1,122 @@ +/* + * ***** BEGIN GPL LICENSE BLOCK ***** + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2011 Blender Foundation. + * All rights reserved. + * + * Contributor(s): Blender Foundation, + * Sergey Sharybin + * + * ***** END GPL LICENSE BLOCK ***** + */ + +// TODO(serrgey): For the time being we're converting simple pipeline +// to an autotrack pipeline we call it tracks. +// Once we've done with porting we remove N. + +#ifndef LIBMV_C_API_TRACKSN_H_ +#define LIBMV_C_API_TRACKSN_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct libmv_TracksN libmv_TracksN; + +// Keep order in this enums exactly the same as in mv::Marker. +// Otherwise API wouldn't convert the values properly. +typedef enum libmv_MarkerSource { + LIBMV_MARKER_SOURCE_MANUAL, + LIBMV_MARKER_SOURCE_DETECTED, + LIBMV_MARKER_SOURCE_TRACKED, + LIBMV_MARKER_SOURCE_MATCHED, + LIBMV_MARKER_SOURCE_PREDICTED, +} libmv_MarkerSource; + +typedef enum libmv_MarkerStatus { + LIBMV_MARKER_STATUS_UNKNOWN, + LIBMV_MARKER_STATUS_INLIER, + LIBMV_MARKER_STATUS_OUTLIER, +} libmv_MarkerStatus; + +typedef enum libmv_MarkerModelType { + LIBMV_MARKER_MODEL_TYPE_POINT, + LIBMV_MARKER_MODEL_TYPE_PLANE, + LIBMV_MARKER_MODEL_TYPE_LINE, + LIBMV_MARKER_MODEL_TYPE_CUBE, +} libmv_MarkerModelType; + +typedef struct libmv_Marker { + int clip; + int frame; + int track; + float center[2]; + float patch[4][2]; + float search_region_min[2]; + float search_region_max[2]; + float weight; + libmv_MarkerSource source; + libmv_MarkerStatus status; + int reference_clip; + int reference_frame; + libmv_MarkerModelType model_type; + int model_id; +} libmv_Marker; + +#ifdef __cplusplus +namespace mv { + class Marker; +} +void libmv_apiMarkerToMarker(const libmv_Marker& libmv_marker, + mv::Marker *marker); + +void libmv_markerToApiMarker(const mv::Marker& marker, + libmv_Marker *libmv_marker); +#endif + +libmv_TracksN* libmv_tracksNewN(void); + +void libmv_tracksDestroyN(libmv_TracksN* libmv_tracks); + + +void libmv_tracksAddMarkerN(libmv_TracksN* libmv_tracks, + const libmv_Marker* libmv_marker); + +void libmv_tracksGetMarkerN(libmv_TracksN* libmv_tracks, + int clip, + int frame, + int track, + libmv_Marker* libmv_marker); + +void libmv_tracksRemoveMarkerN(libmv_TracksN* libmv_tracks, + int clip, + int frame, + int track); + +void libmv_tracksRemoveMarkersForTrack(libmv_TracksN* libmv_tracks, + int track); + +int libmv_tracksMaxClipN(libmv_TracksN* libmv_tracks); +int libmv_tracksMaxFrameN(libmv_TracksN* libmv_tracks, int clip); +int libmv_tracksMaxTrackN(libmv_TracksN* libmv_tracks); +int libmv_tracksNumMarkersN(libmv_TracksN* libmv_tracks); + +#ifdef __cplusplus +} +#endif + +#endif // LIBMV_C_API_TRACKS_H_ diff --git a/extern/libmv/libmv-capi.h b/extern/libmv/libmv-capi.h index 524f1822bc7..92e206a19b2 100644 --- a/extern/libmv/libmv-capi.h +++ b/extern/libmv/libmv-capi.h @@ -27,13 +27,16 @@ #ifndef LIBMV_C_API_H #define LIBMV_C_API_H +#include "intern/autotrack.h" #include "intern/camera_intrinsics.h" #include "intern/detector.h" +#include "intern/frame_accessor.h" #include "intern/homography.h" #include "intern/image.h" #include "intern/logging.h" #include "intern/reconstruction.h" #include "intern/track_region.h" #include "intern/tracks.h" +#include "intern/tracksN.h" #endif // LIBMV_C_API_H diff --git a/extern/libmv/libmv/autotrack/autotrack.cc b/extern/libmv/libmv/autotrack/autotrack.cc new file mode 100644 index 00000000000..ea5e2f7a8db --- /dev/null +++ b/extern/libmv/libmv/autotrack/autotrack.cc @@ -0,0 +1,244 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#include "libmv/autotrack/autotrack.h" +#include "libmv/autotrack/quad.h" +#include "libmv/autotrack/frame_accessor.h" +#include "libmv/autotrack/predict_tracks.h" +#include "libmv/logging/logging.h" +#include "libmv/numeric/numeric.h" + +namespace mv { + +namespace { + +template<typename QuadT, typename ArrayT> +void QuadToArrays(const QuadT& quad, ArrayT* x, ArrayT* y) { + for (int i = 0; i < 4; ++i) { + x[i] = quad.coordinates(i, 0); + y[i] = quad.coordinates(i, 1); + } +} + +void MarkerToArrays(const Marker& marker, double* x, double* y) { + Quad2Df offset_quad = marker.patch; + Vec2f origin = marker.search_region.Rounded().min; + offset_quad.coordinates.rowwise() -= origin.transpose(); + QuadToArrays(offset_quad, x, y); + x[4] = marker.center.x() - origin(0); + y[4] = marker.center.y() - origin(1); +} + +FrameAccessor::Key GetImageForMarker(const Marker& marker, + FrameAccessor* frame_accessor, + FloatImage* image) { + // TODO(sergey): Currently we pass float region to the accessor, + // but we don't want the accessor to decide the rounding, so we + // do rounding here. + // Ideally we would need to pass IntRegion to the frame accessor. + Region region = marker.search_region.Rounded(); + return frame_accessor->GetImage(marker.clip, + marker.frame, + FrameAccessor::MONO, + 0, // No downscale for now. + ®ion, + NULL, + image); +} + +} // namespace + +bool AutoTrack::TrackMarker(Marker* tracked_marker, + TrackRegionResult* result, + const TrackRegionOptions* track_options) { + // Try to predict the location of the second marker. + bool predicted_position = false; + if (PredictMarkerPosition(tracks_, tracked_marker)) { + LG << "Succesfully predicted!"; + predicted_position = true; + } else { + LG << "Prediction failed; trying to track anyway."; + } + + Marker reference_marker; + tracks_.GetMarker(tracked_marker->reference_clip, + tracked_marker->reference_frame, + tracked_marker->track, + &reference_marker); + + // Convert markers into the format expected by TrackRegion. + double x1[5], y1[5]; + MarkerToArrays(reference_marker, x1, y1); + + double x2[5], y2[5]; + MarkerToArrays(*tracked_marker, x2, y2); + + // TODO(keir): Technically this could take a smaller slice from the source + // image instead of taking one the size of the search window. + FloatImage reference_image; + FrameAccessor::Key reference_key = GetImageForMarker(reference_marker, + frame_accessor_, + &reference_image); + if (!reference_key) { + LG << "Couldn't get frame for reference marker: " << reference_marker; + return false; + } + + FloatImage tracked_image; + FrameAccessor::Key tracked_key = GetImageForMarker(*tracked_marker, + frame_accessor_, + &tracked_image); + if (!tracked_key) { + LG << "Couldn't get frame for tracked marker: " << tracked_marker; + return false; + } + + // Store original position befoer tracking, so we can claculate offset later. + Vec2f original_center = tracked_marker->center; + + // Do the tracking! + TrackRegionOptions local_track_region_options; + if (track_options) { + local_track_region_options = *track_options; + } + local_track_region_options.num_extra_points = 1; // For center point. + local_track_region_options.attempt_refine_before_brute = predicted_position; + TrackRegion(reference_image, + tracked_image, + x1, y1, + local_track_region_options, + x2, y2, + result); + + // Copy results over the tracked marker. + Vec2f tracked_origin = tracked_marker->search_region.Rounded().min; + for (int i = 0; i < 4; ++i) { + tracked_marker->patch.coordinates(i, 0) = x2[i] + tracked_origin[0]; + tracked_marker->patch.coordinates(i, 1) = y2[i] + tracked_origin[1]; + } + tracked_marker->center(0) = x2[4] + tracked_origin[0]; + tracked_marker->center(1) = y2[4] + tracked_origin[1]; + Vec2f delta = tracked_marker->center - original_center; + tracked_marker->search_region.Offset(delta); + tracked_marker->source = Marker::TRACKED; + tracked_marker->status = Marker::UNKNOWN; + tracked_marker->reference_clip = reference_marker.clip; + tracked_marker->reference_frame = reference_marker.frame; + + // Release the images from the accessor cache. + frame_accessor_->ReleaseImage(reference_key); + frame_accessor_->ReleaseImage(tracked_key); + + // TODO(keir): Possibly the return here should get removed since the results + // are part of TrackResult. However, eventually the autotrack stuff will have + // extra status (e.g. prediction fail, etc) that should get included. + return true; +} + +void AutoTrack::AddMarker(const Marker& marker) { + tracks_.AddMarker(marker); +} + +void AutoTrack::SetMarkers(vector<Marker>* markers) { + tracks_.SetMarkers(markers); +} + +bool AutoTrack::GetMarker(int clip, int frame, int track, + Marker* markers) const { + return tracks_.GetMarker(clip, frame, track, markers); +} + +void AutoTrack::DetectAndTrack(const DetectAndTrackOptions& options) { + int num_clips = frame_accessor_->NumClips(); + for (int clip = 0; clip < num_clips; ++clip) { + int num_frames = frame_accessor_->NumFrames(clip); + vector<Marker> previous_frame_markers; + // Q: How to decide track #s when detecting? + // Q: How to match markers from previous frame? set of prev frame tracks? + // Q: How to decide what markers should get tracked and which ones should not? + for (int frame = 0; frame < num_frames; ++frame) { + if (Cancelled()) { + LG << "Got cancel message while detecting and tracking..."; + return; + } + // First, get or detect markers for this frame. + vector<Marker> this_frame_markers; + tracks_.GetMarkersInFrame(clip, frame, &this_frame_markers); + LG << "Clip " << clip << ", frame " << frame << " have " + << this_frame_markers.size(); + if (this_frame_markers.size() < options.min_num_features) { + DetectFeaturesInFrame(clip, frame); + this_frame_markers.clear(); + tracks_.GetMarkersInFrame(clip, frame, &this_frame_markers); + LG << "... detected " << this_frame_markers.size() << " features."; + } + if (previous_frame_markers.empty()) { + LG << "First frame; skipping tracking stage."; + previous_frame_markers.swap(this_frame_markers); + continue; + } + // Second, find tracks that should get tracked forward into this frame. + // To avoid tracking markers that are already tracked to this frame, make + // a sorted set of the tracks that exist in the last frame. + vector<int> tracks_in_this_frame; + for (int i = 0; i < this_frame_markers.size(); ++i) { + tracks_in_this_frame.push_back(this_frame_markers[i].track); + } + std::sort(tracks_in_this_frame.begin(), + tracks_in_this_frame.end()); + + // Find tracks in the previous frame that are not in this one. + vector<Marker*> previous_frame_markers_to_track; + int num_skipped = 0; + for (int i = 0; i < previous_frame_markers.size(); ++i) { + if (std::binary_search(tracks_in_this_frame.begin(), + tracks_in_this_frame.end(), + previous_frame_markers[i].track)) { + num_skipped++; + } else { + previous_frame_markers_to_track.push_back(&previous_frame_markers[i]); + } + } + + // Finally track the markers from the last frame into this one. + // TODO(keir): Use OMP. + for (int i = 0; i < previous_frame_markers_to_track.size(); ++i) { + Marker this_frame_marker = *previous_frame_markers_to_track[i]; + this_frame_marker.frame = frame; + LG << "Tracking: " << this_frame_marker; + TrackRegionResult result; + TrackMarker(&this_frame_marker, &result); + if (result.is_usable()) { + LG << "Success: " << this_frame_marker; + AddMarker(this_frame_marker); + this_frame_markers.push_back(this_frame_marker); + } else { + LG << "Failed to track: " << this_frame_marker; + } + } + // Put the markers from this frame + previous_frame_markers.swap(this_frame_markers); + } + } +} + +} // namespace mv diff --git a/extern/libmv/libmv/autotrack/autotrack.h b/extern/libmv/libmv/autotrack/autotrack.h new file mode 100644 index 00000000000..7f8dbb91d98 --- /dev/null +++ b/extern/libmv/libmv/autotrack/autotrack.h @@ -0,0 +1,226 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#ifndef LIBMV_AUTOTRACK_AUTOTRACK_H_ +#define LIBMV_AUTOTRACK_AUTOTRACK_H_ + +#include "libmv/autotrack/tracks.h" +#include "libmv/autotrack/region.h" +#include "libmv/tracking/track_region.h" + +namespace libmv { +class CameraIntrinsics; +}; + +namespace mv { + +using libmv::CameraIntrinsics; +using libmv::TrackRegionOptions; +using libmv::TrackRegionResult; + +class FrameAccessor; +class OperationListener; + +// The coordinator of all tracking operations; keeps track of all state +// relating to tracking and reconstruction; for example, 2D tracks and motion +// models, reconstructed cameras, points, and planes; tracking settings; etc. +// +// Typical usage for full autotrack: +// +// AutoTrack auto_track(image_accessor); +// auto_track.SetNumFramesInClip(0, 10); +// auto_track.SetNumFramesInClip(1, 54); +// auto_track.AutoTrack() +// +// It is also possible to specify options to control the reconstruction. +// Furthermore, the individual methods of reconstruction are exposed to make it +// possible to interact with the pipeline as it runs. For example, to track one +// marker across frames, +// +// AutoTrack auto_track(image_accessor); +// auto_track.SetNumFramesInClip(0, 10); +// auto_track.SetNumFramesInClip(1, 54); +// auto_track.AddMarker(...); +// auto_track.TrackMarkerToFrame(int clip1, int frame1, +// int clip2, int frame2, +// options?) +// +class AutoTrack { + public: + struct Options { + // Default configuration for 2D tracking when calling TrackMarkerToFrame(). + TrackRegionOptions track_region; + + // Default search window for region tracking, in absolute frame pixels. + Region search_region; + }; + + AutoTrack(FrameAccessor* frame_accessor) + : frame_accessor_(frame_accessor) {} + + // Marker manipulation. + // Clip manipulation. + + // Set the number of clips. These clips will get accessed from the frame + // accessor, matches between frames found, and a reconstruction created. + //void SetNumFrames(int clip, int num_frames); + + // Tracking & Matching + + // Find the marker for the track in the frame indicated by the marker. + // Caller maintains ownership of *result and *tracked_marker. + bool TrackMarker(Marker* tracked_marker, + TrackRegionResult* result, + const TrackRegionOptions* track_options=NULL); + + // Wrapper around Tracks API; however these may add additional processing. + void AddMarker(const Marker& tracked_marker); + void SetMarkers(vector<Marker>* markers); + bool GetMarker(int clip, int frame, int track, Marker* marker) const; + + // TODO(keir): Implement frame matching! This could be very cool for loop + // closing and connecting across clips. + //void MatchFrames(int clip1, int frame1, int clip2, int frame2) {} + + // Wrapper around the Reconstruction API. + // Returns the new ID. + int AddCameraIntrinsics(CameraIntrinsics* intrinsics) { + (void) intrinsics; + return 0; + } // XXX + int SetClipIntrinsics(int clip, int intrinsics) { + (void) clip; + (void) intrinsics; + return 0; + } // XXX + + enum Motion { + GENERAL_CAMERA_MOTION, + TRIPOD_CAMERA_MOTION, + }; + int SetClipMotion(int clip, Motion motion) { + (void) clip; + (void) motion; + return 0; + } // XXX + + // Decide what to refine for the given intrinsics. bundle_options is from + // bundle.h (e.g. BUNDLE_FOCAL_LENGTH | BUNDLE_RADIAL_K1). + void SetIntrinsicsRefine(int intrinsics, int bundle_options) { + (void) intrinsics; + (void) bundle_options; + } // XXX + + // Keyframe read/write. + struct ClipFrame { + int clip; + int frame; + }; + const vector<ClipFrame>& keyframes() { return keyframes_; } + void ClearKeyframes() { keyframes_.clear(); } + void SetKeyframes(const vector<ClipFrame>& keyframes) { + keyframes_ = keyframes; + } + + // What about reporting what happened? -- callbacks; maybe result struct. + void Reconstruct(); + + // Detect and track in 2D. + struct DetectAndTrackOptions { + int min_num_features; + }; + void DetectAndTrack(const DetectAndTrackOptions& options); + + struct DetectFeaturesInFrameOptions { + }; + void DetectFeaturesInFrame(int clip, int frame, + const DetectFeaturesInFrameOptions* options=NULL) { + (void) clip; + (void) frame; + (void) options; + } // XXX + + // Does not take ownership of the given listener, but keeps a reference to it. + void AddListener(OperationListener* listener) {(void) listener;} // XXX + + // Create the initial reconstruction, + //void FindInitialReconstruction(); + + // State machine + // + // Question: Have explicit state? Or determine state from existing data? + // Conclusion: Determine state from existing data. + // + // Preliminary state thoughts + // + // No tracks or markers + // - Tracks empty. + // + // Initial tracks found + // - All images have at least 5 tracks + // + // Ran RANSAC on tracks to mark inliers / outliers. + // - All images have at least 8 "inlier" tracks + // + // Detector matching run to close loops and match across clips + // - At least 5 matching tracks between clips + // + // Initial reconstruction found (2 frames)? + // - There exists two cameras with intrinsics / extrinsics + // + // Preliminary reconstruction finished + // - Poses for all frames in all clips estimated. + // + // Final reconstruction finished + // - Final reconstruction bundle adjusted. + + // For now, expose options directly. In the future this may change. + Options options; + + private: + bool Log(); + bool Progress(); + bool Cancelled() { return false; } + + Tracks tracks_; // May be normalized camera coordinates or raw pixels. + //Reconstruction reconstruction_; + + // TODO(keir): Add the motion models here. + //vector<MotionModel> motion_models_; + + // TODO(keir): Should num_clips and num_frames get moved to FrameAccessor? + // TODO(keir): What about masking for clips and frames to prevent various + // things like reconstruction or tracking from happening on certain frames? + FrameAccessor* frame_accessor_; + //int num_clips_; + //vector<int> num_frames_; // Indexed by clip. + + // The intrinsics for each clip, assuming each clip has fixed intrinsics. + // TODO(keir): Decide what the semantics should be for varying focal length. + vector<int> clip_intrinsics_; + + vector<ClipFrame> keyframes_; +}; + +} // namespace mv + +#endif // LIBMV_AUTOTRACK_AUTOTRACK_H_ diff --git a/extern/libmv/libmv/autotrack/callbacks.h b/extern/libmv/libmv/autotrack/callbacks.h new file mode 100644 index 00000000000..e65841de3ce --- /dev/null +++ b/extern/libmv/libmv/autotrack/callbacks.h @@ -0,0 +1,38 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#ifndef LIBMV_AUTOTRACK_LISTENER_H_ +#define LIBMV_AUTOTRACK_LISTENER_H_ + +namespace mv { + +struct OperationListener { + // All hooks return true to continue or false to indicate the operation + // should abort. Hooks should be thread safe (reentrant). + virtual bool Log(const string& message) = 0; + virtual bool Progress(double fraction) = 0; + virtual bool Cancelled() = 0; +}; + +} // namespace mv + +#endif // LIBMV_AUTOTRACK_LISTENER_H_ diff --git a/extern/libmv/libmv/autotrack/frame_accessor.h b/extern/libmv/libmv/autotrack/frame_accessor.h new file mode 100644 index 00000000000..e788bb8ab97 --- /dev/null +++ b/extern/libmv/libmv/autotrack/frame_accessor.h @@ -0,0 +1,85 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#ifndef LIBMV_AUTOTRACK_FRAME_ACCESSOR_H_ +#define LIBMV_AUTOTRACK_FRAME_ACCESSOR_H_ + +#include <stdint.h> + +#include "libmv/image/image.h" + +namespace mv { + +struct Region; + +using libmv::FloatImage; + +// This is the abstraction to different sources of images that will be part of +// a reconstruction. These may come from disk or they may come from Blender. In +// most cases it's expected that the implementation provides some caching +// otherwise performance will be terrible. Sometimes the images need to get +// filtered, and this interface provides for that as well (and permits +// implementations to cache filtered image pieces). +struct FrameAccessor { + struct Transform { + // The key should depend on the transform arguments. Must be non-zero. + virtual int64_t key() const = 0; + + // Apply the expected transform. Output is sized correctly already. + // TODO(keir): What about blurs that need to access pixels outside the ROI? + virtual void run(const FloatImage& input, FloatImage* output) const = 0; + }; + + enum InputMode { + MONO, + RGBA + }; + + typedef void* Key; + + // Get a possibly-filtered version of a frame of a video. Downscale will + // cause the input image to get downscaled by 2^downscale for pyramid access. + // Region is always in original-image coordinates, and describes the + // requested area. The transform describes an (optional) transform to apply + // to the image before it is returned. + // + // When done with an image, you must call ReleaseImage with the returned key. + virtual Key GetImage(int clip, + int frame, + InputMode input_mode, + int downscale, // Downscale by 2^downscale. + const Region* region, // Get full image if NULL. + const Transform* transform, // May be NULL. + FloatImage* destination) = 0; + + // Releases an image from the frame accessor. Non-caching implementations may + // free the image immediately; others may hold onto the image. + virtual void ReleaseImage(Key) = 0; + + virtual bool GetClipDimensions(int clip, int* width, int* height) = 0; + virtual int NumClips() = 0; + virtual int NumFrames(int clip) = 0; +}; + +} // namespace libmv + +#endif // LIBMV_AUTOTRACK_FRAME_ACCESSOR_H_ diff --git a/extern/libmv/libmv/autotrack/marker.h b/extern/libmv/libmv/autotrack/marker.h new file mode 100644 index 00000000000..dc73de84ee2 --- /dev/null +++ b/extern/libmv/libmv/autotrack/marker.h @@ -0,0 +1,135 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#ifndef LIBMV_AUTOTRACK_MARKER_H_ +#define LIBMV_AUTOTRACK_MARKER_H_ + +#include <ostream> + +#include "libmv/autotrack/quad.h" +#include "libmv/autotrack/region.h" +#include "libmv/numeric/numeric.h" + +namespace mv { + +using libmv::Vec2f; + +// A marker is the 2D location of a tracked region (quad) in an image. +// Note that some of this information could be normalized by having a +// collection of inter-connected structs. Instead the "fat Marker" design below +// trades memory for data structure simplicity. +struct Marker { + int clip; // The clip this marker is from. + int frame; // The frame within the clip this marker is from. + int track; // The track this marker is from. + + // The center of the marker in frame coordinates. This is typically, but not + // always, the same as the center of the patch. + Vec2f center; + + // A frame-realtive quad defining the part of the image the marker covers. + // For reference markers, the pixels in the patch are the tracking pattern. + Quad2Df patch; + + // Some markers are less certain than others; the weight determines the + // amount this marker contributes to the error. 1.0 indicates normal + // contribution; 0.0 indicates a zero-weight track (and will be omitted from + // bundle adjustment). + float weight; + + enum Source { + MANUAL, // The user placed this marker manually. + DETECTED, // A keypoint detector found this point. + TRACKED, // The tracking algorithm placed this marker. + MATCHED, // A matching algorithm (e.g. SIFT or SURF or ORB) found this. + PREDICTED, // A motion model predicted this marker. This is needed for + // handling occlusions in some cases where an imaginary marker + // is placed to keep camera motion smooth. + }; + Source source; + + // Markers may be inliers or outliers if the tracking fails; this allows + // visualizing the markers in the image. + enum Status { + UNKNOWN, + INLIER, + OUTLIER + }; + Status status; + + // When doing correlation tracking, where to search in the current frame for + // the pattern from the reference frame, in absolute frame coordinates. + Region search_region; + + // For tracked and matched markers, indicates what the reference was. + int reference_clip; + int reference_frame; + + // Model related information for non-point tracks. + // + // Some tracks are on a larger object, such as a plane or a line or perhaps + // another primitive (a rectangular prisim). This captures the information + // needed to say that for example a collection of markers belongs to model #2 + // (and model #2 is a plane). + enum ModelType { + POINT, + PLANE, + LINE, + CUBE + }; + ModelType model_type; + + // The model ID this track (e.g. the second model, which is a plane). + int model_id; + + // TODO(keir): Add a "int model_argument" to capture that e.g. a marker is on + // the 3rd face of a cube. + + // Offset everything (center, patch, search) by the given delta. + template<typename T> + void Offset(const T& offset) { + center += offset.template cast<float>(); + patch.coordinates.rowwise() += offset.template cast<int>(); + search_region.Offset(offset); + } + + // Shift the center to the given new position (and patch, search). + template<typename T> + void SetPosition(const T& new_center) { + Offset(new_center - center); + } +}; + +inline std::ostream& operator<<(std::ostream& out, const Marker& marker) { + out << "{" + << marker.clip << ", " + << marker.frame << ", " + << marker.track << ", (" + << marker.center.x() << ", " + << marker.center.y() << ")" + << "}"; + return out; +} + +} // namespace mv + +#endif // LIBMV_AUTOTRACK_MARKER_H_ diff --git a/extern/libmv/libmv/autotrack/model.h b/extern/libmv/libmv/autotrack/model.h new file mode 100644 index 00000000000..1165281cdac --- /dev/null +++ b/extern/libmv/libmv/autotrack/model.h @@ -0,0 +1,44 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#ifndef LIBMV_AUTOTRACK_MODEL_H_ +#define LIBMV_AUTOTRACK_MODEL_H_ + +#include "libmv/numeric/numeric.h" +#include "libmv/autotrack/quad.h" + +namespace mv { + +struct Model { + enum ModelType { + POINT, + PLANE, + LINE, + CUBE + }; + + // ??? +}; + +} // namespace mv + +#endif // LIBMV_AUTOTRACK_MODEL_H_ diff --git a/extern/libmv/libmv/autotrack/predict_tracks.cc b/extern/libmv/libmv/autotrack/predict_tracks.cc new file mode 100644 index 00000000000..adc986a0033 --- /dev/null +++ b/extern/libmv/libmv/autotrack/predict_tracks.cc @@ -0,0 +1,316 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#include "libmv/autotrack/marker.h" +#include "libmv/autotrack/predict_tracks.h" +#include "libmv/autotrack/tracks.h" +#include "libmv/base/vector.h" +#include "libmv/logging/logging.h" +#include "libmv/tracking/kalman_filter.h" + +namespace mv { + +namespace { + +using libmv::vector; +using libmv::Vec2; + +// Implied time delta between steps. Set empirically by tweaking and seeing +// what numbers did best at prediction. +const double dt = 3.8; + +// State transition matrix. + +// The states for predicting a track are as follows: +// +// 0 - X position +// 1 - X velocity +// 2 - X acceleration +// 3 - Y position +// 4 - Y velocity +// 5 - Y acceleration +// +// Note that in the velocity-only state transition matrix, the acceleration +// component is ignored; so technically the system could be modelled with only +// 4 states instead of 6. For ease of implementation, this keeps order 6. + +// Choose one or the other model from below (velocity or acceleration). + +// For a typical system having constant velocity. This gives smooth-appearing +// predictions, but they are not always as accurate. +const double velocity_state_transition_data[] = { + 1, dt, 0, 0, 0, 0, + 0, 1, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, + 0, 0, 0, 1, dt, 0, + 0, 0, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 1 +}; + +// This 3rd-order system also models acceleration. This makes for "jerky" +// predictions, but that tend to be more accurate. +const double acceleration_state_transition_data[] = { + 1, dt, dt*dt/2, 0, 0, 0, + 0, 1, dt, 0, 0, 0, + 0, 0, 1, 0, 0, 0, + 0, 0, 0, 1, dt, dt*dt/2, + 0, 0, 0, 0, 1, dt, + 0, 0, 0, 0, 0, 1 +}; + +// This system (attempts) to add an angular velocity component. However, it's +// total junk. +const double angular_state_transition_data[] = { + 1, dt, -dt, 0, 0, 0, // Position x + 0, 1, 0, 0, 0, 0, // Velocity x + 0, 0, 1, 0, 0, 0, // Angular momentum + 0, 0, dt, 1, dt, 0, // Position y + 0, 0, 0, 0, 1, 0, // Velocity y + 0, 0, 0, 0, 0, 1 // Ignored +}; + +const double* state_transition_data = velocity_state_transition_data; + +// Observation matrix. +const double observation_data[] = { + 1., 0., 0., 0., 0., 0., + 0., 0., 0., 1., 0., 0. +}; + +// Process covariance. +const double process_covariance_data[] = { + 35, 0, 0, 0, 0, 0, + 0, 5, 0, 0, 0, 0, + 0, 0, 5, 0, 0, 0, + 0, 0, 0, 35, 0, 0, + 0, 0, 0, 0, 5, 0, + 0, 0, 0, 0, 0, 5 +}; + +// Process covariance. +const double measurement_covariance_data[] = { + 0.01, 0.00, + 0.00, 0.01, +}; + +// Initial covariance. +const double initial_covariance_data[] = { + 10, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, + 0, 0, 0, 10, 0, 0, + 0, 0, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 1 +}; + +typedef mv::KalmanFilter<double, 6, 2> TrackerKalman; + +TrackerKalman filter(state_transition_data, + observation_data, + process_covariance_data, + measurement_covariance_data); + +bool OrderByFrameLessThan(const Marker* a, const Marker* b) { + if (a->frame == b->frame) { + if (a->clip == b->clip) { + return a->track < b->track; + } + return a->clip < b->clip; + } + return a->frame < b-> frame; +} + +// Predicted must be after the previous markers (in the frame numbering sense). +void RunPrediction(const vector<Marker*> previous_markers, + Marker* predicted_marker) { + TrackerKalman::State state; + state.mean << previous_markers[0]->center.x(), 0, 0, + previous_markers[0]->center.y(), 0, 0; + state.covariance = Eigen::Matrix<double, 6, 6, Eigen::RowMajor>( + initial_covariance_data); + + int current_frame = previous_markers[0]->frame; + int target_frame = predicted_marker->frame; + + bool predict_forward = current_frame < target_frame; + int frame_delta = predict_forward ? 1 : -1; + + for (int i = 1; i < previous_markers.size(); ++i) { + // Step forward predicting the state until it is on the current marker. + int predictions = 0; + for (; + current_frame != previous_markers[i]->frame; + current_frame += frame_delta) { + filter.Step(&state); + predictions++; + LG << "Predicted point (frame " << current_frame << "): " + << state.mean(0) << ", " << state.mean(3); + } + // Log the error -- not actually used, but interesting. + Vec2 error = previous_markers[i]->center.cast<double>() - + Vec2(state.mean(0), state.mean(3)); + LG << "Prediction error for " << predictions << " steps: (" + << error.x() << ", " << error.y() << "); norm: " << error.norm(); + // Now that the state is predicted in the current frame, update the state + // based on the measurement from the current frame. + filter.Update(previous_markers[i]->center.cast<double>(), + Eigen::Matrix<double, 2, 2, Eigen::RowMajor>( + measurement_covariance_data), + &state); + LG << "Updated point: " << state.mean(0) << ", " << state.mean(3); + } + // At this point as all the prediction that's possible is done. Finally + // predict until the target frame. + for (; current_frame != target_frame; current_frame += frame_delta) { + filter.Step(&state); + LG << "Final predicted point (frame " << current_frame << "): " + << state.mean(0) << ", " << state.mean(3); + } + + // The x and y positions are at 0 and 3; ignore acceleration and velocity. + predicted_marker->center.x() = state.mean(0); + predicted_marker->center.y() = state.mean(3); + + // Take the patch from the last marker then shift it to match the prediction. + const Marker& last_marker = *previous_markers[previous_markers.size() - 1]; + predicted_marker->patch = last_marker.patch; + Vec2f delta = predicted_marker->center - last_marker.center; + for (int i = 0; i < 4; ++i) { + predicted_marker->patch.coordinates.row(i) += delta; + } + + // Alter the search area as well so it always corresponds to the center. + predicted_marker->search_region = last_marker.search_region; + predicted_marker->search_region.Offset(delta); +} + +} // namespace + +bool PredictMarkerPosition(const Tracks& tracks, Marker* marker) { + // Get all markers for this clip and track. + vector<Marker> markers; + tracks.GetMarkersForTrackInClip(marker->clip, marker->track, &markers); + + if (markers.empty()) { + LG << "No markers to predict from for " << *marker; + return false; + } + + // Order the markers by frame within the clip. + vector<Marker*> boxed_markers(markers.size()); + for (int i = 0; i < markers.size(); ++i) { + boxed_markers[i] = &markers[i]; + } + std::sort(boxed_markers.begin(), boxed_markers.end(), OrderByFrameLessThan); + + // Find the insertion point for this marker among the returned ones. + int insert_at = -1; // If we find the exact frame + int insert_before = -1; // Otherwise... + for (int i = 0; i < boxed_markers.size(); ++i) { + if (boxed_markers[i]->frame == marker->frame) { + insert_at = i; + break; + } + if (boxed_markers[i]->frame > marker->frame) { + insert_before = i; + break; + } + } + + // Forward starts at the marker or insertion point, and goes forward. + int forward_scan_begin, forward_scan_end; + + // Backward scan starts at the marker or insertion point, and goes backward. + int backward_scan_begin, backward_scan_end; + + // Determine the scanning ranges. + if (insert_at == -1 && insert_before == -1) { + // Didn't find an insertion point except the end. + forward_scan_begin = forward_scan_end = 0; + backward_scan_begin = markers.size() - 1; + backward_scan_end = 0; + } else if (insert_at != -1) { + // Found existing marker; scan before and after it. + forward_scan_begin = insert_at + 1; + forward_scan_end = markers.size() - 1;; + backward_scan_begin = insert_at - 1; + backward_scan_end = 0; + } else { + // Didn't find existing marker but found an insertion point. + forward_scan_begin = insert_before; + forward_scan_end = markers.size() - 1;; + backward_scan_begin = insert_before - 1; + backward_scan_end = 0; + } + + const int num_consecutive_needed = 2; + + if (forward_scan_begin <= forward_scan_end && + forward_scan_end - forward_scan_begin > num_consecutive_needed) { + // TODO(keir): Finish this. + } + + bool predict_forward = false; + if (backward_scan_end <= backward_scan_begin) { + // TODO(keir): Add smarter handling and detecting of consecutive frames! + predict_forward = true; + } + + const int max_frames_to_predict_from = 20; + if (predict_forward) { + if (backward_scan_begin - backward_scan_end < num_consecutive_needed) { + // Not enough information to do a prediction. + LG << "Predicting forward impossible, not enough information"; + return false; + } + LG << "Predicting forward"; + int predict_begin = + std::max(backward_scan_begin - max_frames_to_predict_from, 0); + int predict_end = backward_scan_begin; + vector<Marker*> previous_markers; + for (int i = predict_begin; i <= predict_end; ++i) { + previous_markers.push_back(boxed_markers[i]); + } + RunPrediction(previous_markers, marker); + return true; + } else { + if (forward_scan_end - forward_scan_begin < num_consecutive_needed) { + // Not enough information to do a prediction. + LG << "Predicting backward impossible, not enough information"; + return false; + } + LG << "Predicting backward"; + int predict_begin = + std::min(forward_scan_begin + max_frames_to_predict_from, + forward_scan_end); + int predict_end = forward_scan_begin; + vector<Marker*> previous_markers; + for (int i = predict_begin; i >= predict_end; --i) { + previous_markers.push_back(boxed_markers[i]); + } + RunPrediction(previous_markers, marker); + return false; + } + +} + +} // namespace mv diff --git a/extern/libmv/libmv/autotrack/predict_tracks.h b/extern/libmv/libmv/autotrack/predict_tracks.h new file mode 100644 index 00000000000..9ff13ad2125 --- /dev/null +++ b/extern/libmv/libmv/autotrack/predict_tracks.h @@ -0,0 +1,37 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#ifndef LIBMV_AUTOTRACK_PREDICT_TRACKS_H_ +#define LIBMV_AUTOTRACK_PREDICT_TRACKS_H_ + +namespace mv { + +class Tracks; +class Marker; + +// Predict the position of the given marker, and update it accordingly. The +// existing position will be overwritten. +bool PredictMarkerPosition(const Tracks& tracks, Marker* marker); + +} // namespace mv + +#endif // LIBMV_AUTOTRACK_PREDICT_TRACKS_H_ diff --git a/extern/libmv/libmv/autotrack/predict_tracks_test.cc b/extern/libmv/libmv/autotrack/predict_tracks_test.cc new file mode 100644 index 00000000000..fc90e260d94 --- /dev/null +++ b/extern/libmv/libmv/autotrack/predict_tracks_test.cc @@ -0,0 +1,201 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#include "libmv/autotrack/predict_tracks.h" + +#include "libmv/autotrack/marker.h" +#include "libmv/autotrack/tracks.h" +#include "libmv/logging/logging.h" +#include "testing/testing.h" + +namespace mv { + +void AddMarker(int frame, float x, float y, Tracks* tracks) { + Marker marker; + marker.clip = marker.track = 0; + marker.frame = frame; + marker.center.x() = x; + marker.center.y() = y; + marker.patch.coordinates << x - 1, y - 1, + x + 1, y - 1, + x + 1, y + 1, + x - 1, y + 1; + tracks->AddMarker(marker); +} + +TEST(PredictMarkerPosition, EasyLinearMotion) { + Tracks tracks; + AddMarker(0, 1.0, 0.0, &tracks); + AddMarker(1, 2.0, 5.0, &tracks); + AddMarker(2, 3.0, 10.0, &tracks); + AddMarker(3, 4.0, 15.0, &tracks); + AddMarker(4, 5.0, 20.0, &tracks); + AddMarker(5, 6.0, 25.0, &tracks); + AddMarker(6, 7.0, 30.0, &tracks); + AddMarker(7, 8.0, 35.0, &tracks); + + Marker predicted; + predicted.clip = 0; + predicted.track = 0; + predicted.frame = 8; + + PredictMarkerPosition(tracks, &predicted); + double error = (libmv::Vec2f(9.0, 40.0) - predicted.center).norm(); + LG << "Got error: " << error; + EXPECT_LT(error, 0.1); + + // Check the patch coordinates as well. + double x = 9, y = 40.0; + Quad2Df expected_patch; + expected_patch.coordinates << x - 1, y - 1, + x + 1, y - 1, + x + 1, y + 1, + x - 1, y + 1; + + error = (expected_patch.coordinates - predicted.patch.coordinates).norm(); + LG << "Patch error: " << error; + EXPECT_LT(error, 0.1); +} + +TEST(PredictMarkerPosition, EasyBackwardLinearMotion) { + Tracks tracks; + AddMarker(8, 1.0, 0.0, &tracks); + AddMarker(7, 2.0, 5.0, &tracks); + AddMarker(6, 3.0, 10.0, &tracks); + AddMarker(5, 4.0, 15.0, &tracks); + AddMarker(4, 5.0, 20.0, &tracks); + AddMarker(3, 6.0, 25.0, &tracks); + AddMarker(2, 7.0, 30.0, &tracks); + AddMarker(1, 8.0, 35.0, &tracks); + + Marker predicted; + predicted.clip = 0; + predicted.track = 0; + predicted.frame = 0; + + PredictMarkerPosition(tracks, &predicted); + LG << predicted; + double error = (libmv::Vec2f(9.0, 40.0) - predicted.center).norm(); + LG << "Got error: " << error; + EXPECT_LT(error, 0.1); + + // Check the patch coordinates as well. + double x = 9.0, y = 40.0; + Quad2Df expected_patch; + expected_patch.coordinates << x - 1, y - 1, + x + 1, y - 1, + x + 1, y + 1, + x - 1, y + 1; + + error = (expected_patch.coordinates - predicted.patch.coordinates).norm(); + LG << "Patch error: " << error; + EXPECT_LT(error, 0.1); +} + +TEST(PredictMarkerPosition, TwoFrameGap) { + Tracks tracks; + AddMarker(0, 1.0, 0.0, &tracks); + AddMarker(1, 2.0, 5.0, &tracks); + AddMarker(2, 3.0, 10.0, &tracks); + AddMarker(3, 4.0, 15.0, &tracks); + AddMarker(4, 5.0, 20.0, &tracks); + AddMarker(5, 6.0, 25.0, &tracks); + AddMarker(6, 7.0, 30.0, &tracks); + // Missing frame 7! + + Marker predicted; + predicted.clip = 0; + predicted.track = 0; + predicted.frame = 8; + + PredictMarkerPosition(tracks, &predicted); + double error = (libmv::Vec2f(9.0, 40.0) - predicted.center).norm(); + LG << "Got error: " << error; + EXPECT_LT(error, 0.1); +} + +TEST(PredictMarkerPosition, FourFrameGap) { + Tracks tracks; + AddMarker(0, 1.0, 0.0, &tracks); + AddMarker(1, 2.0, 5.0, &tracks); + AddMarker(2, 3.0, 10.0, &tracks); + AddMarker(3, 4.0, 15.0, &tracks); + // Missing frames 4, 5, 6, 7. + + Marker predicted; + predicted.clip = 0; + predicted.track = 0; + predicted.frame = 8; + + PredictMarkerPosition(tracks, &predicted); + double error = (libmv::Vec2f(9.0, 40.0) - predicted.center).norm(); + LG << "Got error: " << error; + EXPECT_LT(error, 2.0); // Generous error due to larger prediction window. +} + +TEST(PredictMarkerPosition, MultipleGaps) { + Tracks tracks; + AddMarker(0, 1.0, 0.0, &tracks); + AddMarker(1, 2.0, 5.0, &tracks); + AddMarker(2, 3.0, 10.0, &tracks); + // AddMarker(3, 4.0, 15.0, &tracks); // Note the 3-frame gap. + // AddMarker(4, 5.0, 20.0, &tracks); + // AddMarker(5, 6.0, 25.0, &tracks); + AddMarker(6, 7.0, 30.0, &tracks); // Intermediate measurement. + // AddMarker(7, 8.0, 35.0, &tracks); + + Marker predicted; + predicted.clip = 0; + predicted.track = 0; + predicted.frame = 8; + + PredictMarkerPosition(tracks, &predicted); + double error = (libmv::Vec2f(9.0, 40.0) - predicted.center).norm(); + LG << "Got error: " << error; + EXPECT_LT(error, 1.0); // Generous error due to larger prediction window. +} + +TEST(PredictMarkerPosition, MarkersInRandomOrder) { + Tracks tracks; + + // This is the same as the easy, except that the tracks are randomly ordered. + AddMarker(0, 1.0, 0.0, &tracks); + AddMarker(2, 3.0, 10.0, &tracks); + AddMarker(7, 8.0, 35.0, &tracks); + AddMarker(5, 6.0, 25.0, &tracks); + AddMarker(4, 5.0, 20.0, &tracks); + AddMarker(3, 4.0, 15.0, &tracks); + AddMarker(6, 7.0, 30.0, &tracks); + AddMarker(1, 2.0, 5.0, &tracks); + + Marker predicted; + predicted.clip = 0; + predicted.track = 0; + predicted.frame = 8; + + PredictMarkerPosition(tracks, &predicted); + double error = (libmv::Vec2f(9.0, 40.0) - predicted.center).norm(); + LG << "Got error: " << error; + EXPECT_LT(error, 0.1); +} + +} // namespace mv diff --git a/extern/libmv/libmv/autotrack/quad.h b/extern/libmv/libmv/autotrack/quad.h new file mode 100644 index 00000000000..0c70f9882da --- /dev/null +++ b/extern/libmv/libmv/autotrack/quad.h @@ -0,0 +1,57 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#ifndef LIBMV_AUTOTRACK_QUAD_H_ +#define LIBMV_AUTOTRACK_QUAD_H_ + +#include <Eigen/Core> + +namespace mv { + +template<typename T, int D> +struct Quad { + // A quad is 4 points; generally in 2D or 3D. + // + // +----------> x + // |\. + // | \. + // | z (z goes into screen) + // | + // | r0----->r1 + // | ^ | + // | | . | + // | | V + // | r3<-----r2 + // | \. + // | \. + // v normal goes away (right handed). + // y + // + // Each row is one of the corners coordinates; either (x, y) or (x, y, z). + Eigen::Matrix<T, 4, D> coordinates; +}; + +typedef Quad<float, 2> Quad2Df; + +} // namespace mv + +#endif // LIBMV_AUTOTRACK_QUAD_H_ diff --git a/extern/libmv/libmv/autotrack/reconstruction.h b/extern/libmv/libmv/autotrack/reconstruction.h new file mode 100644 index 00000000000..e1d4e882cbd --- /dev/null +++ b/extern/libmv/libmv/autotrack/reconstruction.h @@ -0,0 +1,89 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#ifndef LIBMV_AUTOTRACK_RECONSTRUCTION_H_ +#define LIBMV_AUTOTRACK_RECONSTRUCTION_H_ + +#include "libmv/base/vector.h" +#include "libmv/numeric/numeric.h" +#include "libmv/simple_pipeline/camera_intrinsics.h" + +namespace mv { + +using libmv::CameraIntrinsics; +using libmv::vector; + +class Model; + +class CameraPose { + int clip; + int frame; + int intrinsics; + Mat3 R; + Vec3 t; +}; + +class Point { + int track; + + // The coordinates of the point. Note that not all coordinates are always + // used; for example points on a plane only use the first two coordinates. + Vec3 X; +}; + +// A reconstruction for a set of tracks. The indexing for clip, frame, and +// track should match that of a Tracs object, stored elsewhere. +class Reconstruction { + public: + // All methods copy their input reference or take ownership of the pointer. + void AddCameraPose(const CameraPose& pose); + int AddCameraIntrinsics(CameraIntrinsics* intrinsics); + int AddPoint(const Point& point); + int AddModel(Model* model); + + // Returns the corresponding pose or point or NULL if missing. + CameraPose* CameraPoseForFrame(int clip, int frame); + const CameraPose* CameraPoseForFrame(int clip, int frame) const; + Point* PointForTrack(int track); + const Point* PointForTrack(int track) const; + + const vector<vector<CameraPose> >& camera_poses() const { + return camera_poses_; + } + + private: + // Indexed by CameraPose::intrinsics. Owns the intrinsics objects. + vector<CameraIntrinsics*> camera_intrinsics_; + + // Indexed by Marker::clip then by Marker::frame. + vector<vector<CameraPose> > camera_poses_; + + // Indexed by Marker::track. + vector<Point> points_; + + // Indexed by Marker::model_id. Owns model objects. + vector<Model*> models_; +}; + +} // namespace mv + +#endif // LIBMV_AUTOTRACK_RECONSTRUCTION_H_ diff --git a/extern/libmv/libmv/autotrack/region.h b/extern/libmv/libmv/autotrack/region.h new file mode 100644 index 00000000000..b35d99eb60d --- /dev/null +++ b/extern/libmv/libmv/autotrack/region.h @@ -0,0 +1,67 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#ifndef LIBMV_AUTOTRACK_REGION_H_ +#define LIBMV_AUTOTRACK_REGION_H_ + +#include "libmv/numeric/numeric.h" + +namespace mv { + +using libmv::Vec2f; + +// A region is a bounding box within an image. +// +// +----------> x +// | +// | (min.x, min.y) (max.x, min.y) +// | +-------------------------+ +// | | | +// | | | +// | | | +// | +-------------------------+ +// v (min.x, max.y) (max.x, max.y) +// y +// +struct Region { + Vec2f min; + Vec2f max; + + template<typename T> + void Offset(const T& offset) { + min += offset.template cast<float>(); + max += offset.template cast<float>(); + } + + Region Rounded() const { + Region result; + result.min(0) = ceil(this->min(0)); + result.min(1) = ceil(this->min(1)); + result.max(0) = ceil(this->max(0)); + result.max(1) = ceil(this->max(1)); + return result; + } +}; + +} // namespace mv + +#endif // LIBMV_AUTOTRACK_REGION_H_ diff --git a/extern/libmv/libmv/autotrack/tracks.cc b/extern/libmv/libmv/autotrack/tracks.cc new file mode 100644 index 00000000000..174f264f3f2 --- /dev/null +++ b/extern/libmv/libmv/autotrack/tracks.cc @@ -0,0 +1,193 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#include "libmv/autotrack/tracks.h" + +#include <algorithm> +#include <vector> +#include <iterator> + +#include "libmv/numeric/numeric.h" + +namespace mv { + +Tracks::Tracks(const Tracks& other) { + markers_ = other.markers_; +} + +Tracks::Tracks(const vector<Marker>& markers) : markers_(markers) {} + +bool Tracks::GetMarker(int clip, int frame, int track, Marker* marker) const { + for (int i = 0; i < markers_.size(); ++i) { + if (markers_[i].clip == clip && + markers_[i].frame == frame && + markers_[i].track == track) { + *marker = markers_[i]; + return true; + } + } + return false; +} + +void Tracks::GetMarkersForTrack(int track, vector<Marker>* markers) const { + for (int i = 0; i < markers_.size(); ++i) { + if (track == markers_[i].track) { + markers->push_back(markers_[i]); + } + } +} + +void Tracks::GetMarkersForTrackInClip(int clip, + int track, + vector<Marker>* markers) const { + for (int i = 0; i < markers_.size(); ++i) { + if (clip == markers_[i].clip && + track == markers_[i].track) { + markers->push_back(markers_[i]); + } + } +} + +void Tracks::GetMarkersInFrame(int clip, + int frame, + vector<Marker>* markers) const { + for (int i = 0; i < markers_.size(); ++i) { + if (markers_[i].clip == clip && + markers_[i].frame == frame) { + markers->push_back(markers_[i]); + } + } +} + +void Tracks::GetMarkersForTracksInBothImages(int clip1, int frame1, + int clip2, int frame2, + vector<Marker>* markers) const { + std::vector<int> image1_tracks; + std::vector<int> image2_tracks; + + // Collect the tracks in each of the two images. + for (int i = 0; i < markers_.size(); ++i) { + int clip = markers_[i].clip; + int frame = markers_[i].frame; + if (clip == clip1 && frame == frame1) { + image1_tracks.push_back(markers_[i].track); + } else if (clip == clip2 && frame == frame2) { + image2_tracks.push_back(markers_[i].track); + } + } + + // Intersect the two sets to find the tracks of interest. + std::sort(image1_tracks.begin(), image1_tracks.end()); + std::sort(image2_tracks.begin(), image2_tracks.end()); + std::vector<int> intersection; + std::set_intersection(image1_tracks.begin(), image1_tracks.end(), + image2_tracks.begin(), image2_tracks.end(), + std::back_inserter(intersection)); + + // Scan through and get the relevant tracks from the two images. + for (int i = 0; i < markers_.size(); ++i) { + // Save markers that are in either frame and are in our candidate set. + if (((markers_[i].clip == clip1 && + markers_[i].frame == frame1) || + (markers_[i].clip == clip2 && + markers_[i].frame == frame2)) && + std::binary_search(intersection.begin(), + intersection.end(), + markers_[i].track)) { + markers->push_back(markers_[i]); + } + } +} + +void Tracks::AddMarker(const Marker& marker) { + // TODO(keir): This is quadratic for repeated insertions. Fix this by adding + // a smarter data structure like a set<>. + for (int i = 0; i < markers_.size(); ++i) { + if (markers_[i].clip == marker.clip && + markers_[i].frame == marker.frame && + markers_[i].track == marker.track) { + markers_[i] = marker; + return; + } + } + markers_.push_back(marker); +} + +void Tracks::SetMarkers(vector<Marker>* markers) { + std::swap(markers_, *markers); +} + +bool Tracks::RemoveMarker(int clip, int frame, int track) { + int size = markers_.size(); + for (int i = 0; i < markers_.size(); ++i) { + if (markers_[i].clip == clip && + markers_[i].frame == frame && + markers_[i].track == track) { + markers_[i] = markers_[size - 1]; + markers_.resize(size - 1); + return true; + } + } + return false; +} + +void Tracks::RemoveMarkersForTrack(int track) { + int size = 0; + for (int i = 0; i < markers_.size(); ++i) { + if (markers_[i].track != track) { + markers_[size++] = markers_[i]; + } + } + markers_.resize(size); +} + +int Tracks::MaxClip() const { + int max_clip = 0; + for (int i = 0; i < markers_.size(); ++i) { + max_clip = std::max(markers_[i].clip, max_clip); + } + return max_clip; +} + +int Tracks::MaxFrame(int clip) const { + int max_frame = 0; + for (int i = 0; i < markers_.size(); ++i) { + if (markers_[i].clip == clip) { + max_frame = std::max(markers_[i].frame, max_frame); + } + } + return max_frame; +} + +int Tracks::MaxTrack() const { + int max_track = 0; + for (int i = 0; i < markers_.size(); ++i) { + max_track = std::max(markers_[i].track, max_track); + } + return max_track; +} + +int Tracks::NumMarkers() const { + return markers_.size(); +} + +} // namespace mv diff --git a/extern/libmv/libmv/autotrack/tracks.h b/extern/libmv/libmv/autotrack/tracks.h new file mode 100644 index 00000000000..0b7de91d211 --- /dev/null +++ b/extern/libmv/libmv/autotrack/tracks.h @@ -0,0 +1,82 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#ifndef LIBMV_AUTOTRACK_TRACKS_H_ +#define LIBMV_AUTOTRACK_TRACKS_H_ + +#include "libmv/base/vector.h" +#include "libmv/autotrack/marker.h" + +namespace mv { + +using libmv::vector; + +// The Tracks container stores correspondences between frames. +class Tracks { + public: + Tracks() { } + Tracks(const Tracks &other); + + // Create a tracks object with markers already initialized. Copies markers. + explicit Tracks(const vector<Marker>& markers); + + // All getters append to the output argument vector. + bool GetMarker(int clip, int frame, int track, Marker* marker) const; + void GetMarkersForTrack(int track, vector<Marker>* markers) const; + void GetMarkersForTrackInClip(int clip, + int track, + vector<Marker>* markers) const; + void GetMarkersInFrame(int clip, int frame, vector<Marker>* markers) const; + + // Get the markers in frame1 and frame2 which have a common track. + // + // This is not the same as the union of the markers in frame1 and + // frame2; each marker is for a track that appears in both images. + void GetMarkersForTracksInBothImages(int clip1, int frame1, + int clip2, int frame2, + vector<Marker>* markers) const; + + void AddMarker(const Marker& marker); + + // Moves the contents of *markers over top of the existing markers. This + // destroys *markers in the process (but avoids copies). + void SetMarkers(vector<Marker>* markers); + bool RemoveMarker(int clip, int frame, int track); + void RemoveMarkersForTrack(int track); + + int MaxClip() const; + int MaxFrame(int clip) const; + int MaxTrack() const; + int NumMarkers() const; + + const vector<Marker>& markers() const { return markers_; } + + private: + vector<Marker> markers_; + + // TODO(keir): Consider adding access-map data structures to avoid all the + // linear lookup penalties for the accessors. +}; + +} // namespace mv + +#endif // LIBMV_AUTOTRACK_TRACKS_H_ diff --git a/extern/libmv/libmv/autotrack/tracks_test.cc b/extern/libmv/libmv/autotrack/tracks_test.cc new file mode 100644 index 00000000000..028b4a10913 --- /dev/null +++ b/extern/libmv/libmv/autotrack/tracks_test.cc @@ -0,0 +1,52 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: mierle@gmail.com (Keir Mierle) + +#include "libmv/autotrack/tracks.h" + +#include "testing/testing.h" +#include "libmv/logging/logging.h" + +namespace mv { + +TEST(Tracks, MaxFrame) { + Marker marker; + Tracks tracks; + + // Add some markers to clip 0. + marker.clip = 0; + marker.frame = 1; + tracks.AddMarker(marker); + + // Add some markers to clip 1. + marker.clip = 1; + marker.frame = 1; + tracks.AddMarker(marker); + + marker.clip = 1; + marker.frame = 12; + tracks.AddMarker(marker); + + EXPECT_EQ(1, tracks.MaxFrame(0)); + EXPECT_EQ(12, tracks.MaxFrame(1)); +} + +} // namespace mv diff --git a/extern/libmv/libmv/base/vector.h b/extern/libmv/libmv/base/vector.h index 067db4ba7f7..1931fb0b1f9 100644 --- a/extern/libmv/libmv/base/vector.h +++ b/extern/libmv/libmv/base/vector.h @@ -101,8 +101,6 @@ class vector { size_ = size; } - - void push_back(const T &value) { if (size_ == capacity_) { reserve(size_ ? 2 * size_ : 1); @@ -130,6 +128,10 @@ class vector { } } + bool empty() { + return size_ == 0; + } + private: void construct(int start, int end) { for (int i = start; i < end; ++i) { diff --git a/extern/libmv/libmv/image/array_nd.h b/extern/libmv/libmv/image/array_nd.h index c5099f24d7b..b56a765223b 100644 --- a/extern/libmv/libmv/image/array_nd.h +++ b/extern/libmv/libmv/image/array_nd.h @@ -41,33 +41,35 @@ class ArrayND : public BaseArray { typedef Tuple<int, N> Index; /// Create an empty array. - ArrayND() : data_(NULL), own_data(true) { Resize(Index(0)); } + ArrayND() : data_(NULL), own_data_(true) { Resize(Index(0)); } /// Create an array with the specified shape. - ArrayND(const Index &shape) : data_(NULL), own_data(true) { Resize(shape); } + ArrayND(const Index &shape) : data_(NULL), own_data_(true) { Resize(shape); } /// Create an array with the specified shape. - ArrayND(int *shape) : data_(NULL), own_data(true) { Resize(shape); } + ArrayND(int *shape) : data_(NULL), own_data_(true) { Resize(shape); } /// Copy constructor. - ArrayND(const ArrayND<T, N> &b) : data_(NULL), own_data(true) { + ArrayND(const ArrayND<T, N> &b) : data_(NULL), own_data_(true) { ResizeLike(b); std::memcpy(Data(), b.Data(), sizeof(T) * Size()); } - ArrayND(int s0) : data_(NULL), own_data(true) { Resize(s0); } - ArrayND(int s0, int s1) : data_(NULL), own_data(true) { Resize(s0, s1); } - ArrayND(int s0, int s1, int s2) : data_(NULL), own_data(true) { + ArrayND(int s0) : data_(NULL), own_data_(true) { Resize(s0); } + ArrayND(int s0, int s1) : data_(NULL), own_data_(true) { Resize(s0, s1); } + ArrayND(int s0, int s1, int s2) : data_(NULL), own_data_(true) { Resize(s0, s1, s2); } - ArrayND(T* data, int s0, int s1, int s2) : data_(data), own_data(false) { + ArrayND(T* data, int s0, int s1, int s2) : data_(data), own_data_(false) { Resize(s0, s1, s2); } /// Destructor deletes pixel data. ~ArrayND() { - delete [] data_; + if (own_data_) { + delete [] data_; + } } /// Assignation copies pixel data. @@ -97,7 +99,7 @@ class ArrayND : public BaseArray { for (int i = N - 1; i > 0; --i) { strides_(i - 1) = strides_(i) * shape_(i); } - if (own_data) { + if (own_data_) { delete [] data_; data_ = NULL; if (Size() > 0) { @@ -336,7 +338,7 @@ class ArrayND : public BaseArray { T *data_; /// Flag if this Array either own or reference the data - bool own_data; + bool own_data_; }; /// 3D array (row, column, channel). diff --git a/extern/libmv/libmv/simple_pipeline/keyframe_selection.cc b/extern/libmv/libmv/simple_pipeline/keyframe_selection.cc index 7b90c28bbca..241b5600505 100644 --- a/extern/libmv/libmv/simple_pipeline/keyframe_selection.cc +++ b/extern/libmv/libmv/simple_pipeline/keyframe_selection.cc @@ -69,54 +69,50 @@ double GRIC(const Vec &e, int d, int k, int r) { // http://www.robots.ox.ac.uk/~vgg/publications/papers/torr99.ps.gz double lambda3 = 2.0; - // measurement error of tracker + // Variance of tracker position. Physically, this is typically about 0.1px, + // and when squared becomes 0.01 px^2. double sigma2 = 0.01; - // Actual GRIC computation - double gric_result = 0.0; - + // Finally, calculate the GRIC score. + double gric = 0.0; for (int i = 0; i < n; i++) { - double rho = std::min(e(i) * e(i) / sigma2, lambda3 * (r - d)); - gric_result += rho; + gric += std::min(e(i) * e(i) / sigma2, lambda3 * (r - d)); } - - gric_result += lambda1 * d * n; - gric_result += lambda2 * k; - - return gric_result; + gric += lambda1 * d * n; + gric += lambda2 * k; + return gric; } -// Compute a generalized inverse using eigen value decomposition. -// It'll actually also zero 7 last eigen values to deal with -// gauges, since this function is used to compute variance of +// Compute a generalized inverse using eigen value decomposition, clamping the +// smallest eigenvalues if requested. This is needed to compute the variance of // reconstructed 3D points. // -// TODO(sergey): Could be generalized by making it so number -// of values to be zeroed is passed by an argument -// and moved to numeric module. -Mat pseudoInverse(const Mat &matrix) { - Eigen::EigenSolver<Mat> eigenSolver(matrix); - Mat D = eigenSolver.pseudoEigenvalueMatrix(); - Mat V = eigenSolver.pseudoEigenvectors(); - +// TODO(keir): Consider moving this into the numeric code, since this is not +// related to keyframe selection. +Mat PseudoInverseWithClampedEigenvalues(const Mat &matrix, + int num_eigenvalues_to_clamp) { + Eigen::EigenSolver<Mat> eigen_solver(matrix); + Mat D = eigen_solver.pseudoEigenvalueMatrix(); + Mat V = eigen_solver.pseudoEigenvectors(); + + // Clamp too-small singular values to zero to prevent numeric blowup. double epsilon = std::numeric_limits<double>::epsilon(); - for (int i = 0; i < D.cols(); ++i) { - if (D(i, i) > epsilon) + if (D(i, i) > epsilon) { D(i, i) = 1.0 / D(i, i); - else + } else { D(i, i) = 0.0; + } } - // Zero last 7 (which corresponds to smallest eigen values). - // 7 equals to the number of gauge freedoms. - for (int i = D.cols() - 7; i < D.cols(); ++i) + // Apply the clamp. + for (int i = D.cols() - num_eigenvalues_to_clamp; i < D.cols(); ++i) { D(i, i) = 0.0; - + } return V * D * V.inverse(); } -void filterZeroWeightMarkersFromTracks(const Tracks &tracks, +void FilterZeroWeightMarkersFromTracks(const Tracks &tracks, Tracks *filtered_tracks) { vector<Marker> all_markers = tracks.AllMarkers(); @@ -143,7 +139,7 @@ void SelectKeyframesBasedOnGRICAndVariance(const Tracks &_tracks, // http://www.cs.ait.ac.th/~mdailey/papers/Tahir-KeyFrame.pdf Tracks filtered_tracks; - filterZeroWeightMarkersFromTracks(_tracks, &filtered_tracks); + FilterZeroWeightMarkersFromTracks(_tracks, &filtered_tracks); int max_image = filtered_tracks.MaxImage(); int next_keyframe = 1; @@ -224,9 +220,9 @@ void SelectKeyframesBasedOnGRICAndVariance(const Tracks &_tracks, EstimateFundamentalOptions estimate_fundamental_options; EstimateFundamentalFromCorrespondences(x1, - x2, - estimate_fundamental_options, - &F); + x2, + estimate_fundamental_options, + &F); // Convert fundamental to original pixel space. F = N_inverse * F * N; @@ -379,7 +375,8 @@ void SelectKeyframesBasedOnGRICAndVariance(const Tracks &_tracks, Mat &jacobian = evaluation.jacobian; Mat JT_J = jacobian.transpose() * jacobian; - Mat JT_J_inv = pseudoInverse(JT_J); + // There are 7 degrees of freedom, so clamp them out. + Mat JT_J_inv = PseudoInverseWithClampedEigenvalues(JT_J, 7); Mat temp_derived = JT_J * JT_J_inv * JT_J; bool is_inversed = (temp_derived - JT_J).cwiseAbs2().sum() < diff --git a/extern/libmv/libmv/tracking/kalman_filter.h b/extern/libmv/libmv/tracking/kalman_filter.h new file mode 100644 index 00000000000..9841f0e912c --- /dev/null +++ b/extern/libmv/libmv/tracking/kalman_filter.h @@ -0,0 +1,112 @@ +// Copyright (c) 2014 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. + +#ifndef LIBMV_TRACKING_KALMAN_FILTER_H_ + +#include "libmv/numeric/numeric.h" + +namespace mv { + +// A Kalman filter with order N and observation size K. +template<typename T, int N, int K> +class KalmanFilter { + public: + struct State { + Eigen::Matrix<T, N, 1> mean; + Eigen::Matrix<T, N, N> covariance; + }; + + // Initialize from row-major data; convenient for constant initializers. + KalmanFilter(const T* state_transition_data, + const T* observation_data, + const T* process_covariance_data, + const T* default_measurement_covariance_data) + : state_transition_matrix_( + Eigen::Matrix<T, N, N, Eigen::RowMajor>(state_transition_data)), + observation_matrix_( + Eigen::Matrix<T, K, N, Eigen::RowMajor>(observation_data)), + process_covariance_( + Eigen::Matrix<T, N, N, Eigen::RowMajor>(process_covariance_data)), + default_measurement_covariance_( + Eigen::Matrix<T, K, K, Eigen::RowMajor>( + default_measurement_covariance_data)) { + } + + KalmanFilter( + const Eigen::Matrix<T, N, N> &state_transition_matrix, + const Eigen::Matrix<T, K, N> &observation_matrix, + const Eigen::Matrix<T, N, N> &process_covariance, + const Eigen::Matrix<T, K, K> &default_measurement_covariance) + : state_transition_matrix_(state_transition_matrix), + observation_matrix_(observation_matrix), + process_covariance_(process_covariance), + default_measurement_covariance_(default_measurement_covariance) { + } + + // Advances the system according to the current state estimate. + void Step(State *state) const { + state->mean = state_transition_matrix_ * state->mean; + state->covariance = state_transition_matrix_ * + state->covariance * + state_transition_matrix_.transpose() + + process_covariance_; + } + + // Updates a state with a new measurement. + void Update(const Eigen::Matrix<T, K, 1> &measurement_mean, + const Eigen::Matrix<T, K, K> &measurement_covariance, + State *state) const { + // Calculate the innovation, which is a distribution over prediction error. + Eigen::Matrix<T, K, 1> innovation_mean = measurement_mean - + observation_matrix_ * + state->mean; + Eigen::Matrix<T, K, K> innovation_covariance = + observation_matrix_ * + state->covariance * + observation_matrix_.transpose() + + measurement_covariance; + + // Calculate the Kalman gain. + Eigen::Matrix<T, 6, 2> kalman_gain = state->covariance * + observation_matrix_.transpose() * + innovation_covariance.inverse(); + + // Update the state mean and covariance. + state->mean += kalman_gain * innovation_mean; + state->covariance = (Eigen::Matrix<T, N, N>::Identity() - + kalman_gain * observation_matrix_) * + state->covariance; + } + + void Update(State *state, + const Eigen::Matrix<T, K, 1> &measurement_mean) const { + Update(state, measurement_mean, default_measurement_covariance_); + } + + private: + const Eigen::Matrix<T, N, N> state_transition_matrix_; + const Eigen::Matrix<T, K, N> observation_matrix_; + const Eigen::Matrix<T, N, N> process_covariance_; + const Eigen::Matrix<T, K, K> default_measurement_covariance_; +}; + +} // namespace mv + +#endif // LIBMV_TRACKING_KALMAN_FILTER_H_ diff --git a/extern/libmv/libmv/tracking/track_region.h b/extern/libmv/libmv/tracking/track_region.h index 2ced9dd6550..be1d8ef3e03 100644 --- a/extern/libmv/libmv/tracking/track_region.h +++ b/extern/libmv/libmv/tracking/track_region.h @@ -42,7 +42,13 @@ struct TrackRegionOptions { }; Mode mode; + // Minimum normalized cross-correlation necessary between the final tracked + // positoin of the patch on the destination image and the reference patch + // needed to declare tracking success. If the minimum correlation is not met, + // then TrackResult::termination is INSUFFICIENT_CORRELATION. double minimum_correlation; + + // Maximum number of Ceres iterations to run for the inner minimization. int max_iterations; // Use the "Efficient Second-order Minimization" scheme. This increases @@ -124,6 +130,11 @@ struct TrackRegionResult { }; Termination termination; + bool is_usable() { + return termination == CONVERGENCE || + termination == NO_CONVERGENCE; + } + int num_iterations; double correlation; diff --git a/source/blender/blenkernel/BKE_tracking.h b/source/blender/blenkernel/BKE_tracking.h index 8bc619c1b27..5f5cb4c17cb 100644 --- a/source/blender/blenkernel/BKE_tracking.h +++ b/source/blender/blenkernel/BKE_tracking.h @@ -210,16 +210,18 @@ void BKE_tracking_disable_channels(struct ImBuf *ibuf, bool disable_red, bool di bool disable_blue, bool grayscale); /* **** 2D tracking **** */ -struct MovieTrackingContext *BKE_tracking_context_new(struct MovieClip *clip, struct MovieClipUser *user, - const bool backwards, const bool sequence); -void BKE_tracking_context_free(struct MovieTrackingContext *context); -void BKE_tracking_context_sync(struct MovieTrackingContext *context); -void BKE_tracking_context_sync_user(const struct MovieTrackingContext *context, struct MovieClipUser *user); -bool BKE_tracking_context_step(struct MovieTrackingContext *context); -void BKE_tracking_context_finish(struct MovieTrackingContext *context); - void BKE_tracking_refine_marker(struct MovieClip *clip, struct MovieTrackingTrack *track, struct MovieTrackingMarker *marker, bool backwards); +/* *** 2D auto track *** */ + +struct AutoTrackContext *BKE_autotrack_context_new(struct MovieClip *clip, struct MovieClipUser *user, + const bool backwards, const bool sequence); +bool BKE_autotrack_context_step(struct AutoTrackContext *context); +void BKE_autotrack_context_sync(struct AutoTrackContext *context); +void BKE_autotrack_context_sync_user(struct AutoTrackContext *context, struct MovieClipUser *user); +void BKE_autotrack_context_finish(struct AutoTrackContext *context); +void BKE_autotrack_context_free(struct AutoTrackContext *context); + /* **** Plane tracking **** */ void BKE_tracking_track_plane_from_existing_motion(struct MovieTrackingPlaneTrack *plane_track, int start_frame); diff --git a/source/blender/blenkernel/CMakeLists.txt b/source/blender/blenkernel/CMakeLists.txt index f4a3a0c10dd..6550882c2f7 100644 --- a/source/blender/blenkernel/CMakeLists.txt +++ b/source/blender/blenkernel/CMakeLists.txt @@ -158,6 +158,7 @@ set(SRC intern/text.c intern/texture.c intern/tracking.c + intern/tracking_auto.c intern/tracking_detect.c intern/tracking_plane_tracker.c intern/tracking_region_tracker.c diff --git a/source/blender/blenkernel/intern/tracking_auto.c b/source/blender/blenkernel/intern/tracking_auto.c new file mode 100644 index 00000000000..2cd13a90048 --- /dev/null +++ b/source/blender/blenkernel/intern/tracking_auto.c @@ -0,0 +1,563 @@ +/* + * ***** BEGIN GPL LICENSE BLOCK ***** + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2011 Blender Foundation. + * All rights reserved. + * + * Contributor(s): Blender Foundation, + * Sergey Sharybin + * Keir Mierle + * + * ***** END GPL LICENSE BLOCK ***** + */ + +/** \file blender/blenkernel/intern/tracking_auto.c + * \ingroup bke + */ + +#include <stdlib.h> + +#include "MEM_guardedalloc.h" + +#include "DNA_movieclip_types.h" +#include "DNA_object_types.h" /* SELECT */ + +#include "BLI_threads.h" +#include "BLI_utildefines.h" +#include "BLI_math.h" + +#include "BKE_movieclip.h" +#include "BKE_tracking.h" + +#include "libmv-capi.h" +#include "tracking_private.h" + +typedef struct AutoTrackOptions { + int clip_index; /** Index of the clip this track belogs to. */ + int track_index; /* Index of the track in AutoTrack tracks structure. */ + MovieTrackingTrack *track; /* Pointer to an original track/ */ + libmv_TrackRegionOptions track_region_options; /* Options for the region + tracker. */ + bool use_keyframe_match; /* Keyframe pattern matching. */ + + /* TODO(sergey): A bit awkward to keep it in here, only used to + * place a disabled marker once the trackign fails, + * Wither find a more clear way to do it or call it track context + * or state, not options. + */ + bool is_failed; + int failed_frame; +} AutoTrackOptions; + +typedef struct AutoTrackContext { + MovieClip *clips[MAX_ACCESSOR_CLIP]; + int num_clips; + + MovieClipUser user; + int frame_width, frame_height; + + struct libmv_AutoTrack *autotrack; + TrackingImageAccessor *image_accessor; + + int num_tracks; /* Number of tracks being tracked. */ + AutoTrackOptions *options; /* Per-tracking track options. */ + + bool backwards; + bool sequence; + int first_frame; + int sync_frame; + bool first_sync; + SpinLock spin_lock; +} AutoTrackContext; + +static void normalized_to_libmv_frame(const float normalized[2], + const int frame_dimensions[2], + float result[2]) +{ + result[0] = normalized[0] * frame_dimensions[0] - 0.5f; + result[1] = normalized[1] * frame_dimensions[1] - 0.5f; +} + +static void normalized_relative_to_libmv_frame(const float normalized[2], + const float origin[2], + const int frame_dimensions[2], + float result[2]) +{ + result[0] = (normalized[0] + origin[0]) * frame_dimensions[0] - 0.5f; + result[1] = (normalized[1] + origin[1]) * frame_dimensions[1] - 0.5f; +} + +static void libmv_frame_to_normalized(const float frame_coord[2], + const int frame_dimensions[2], + float result[2]) +{ + result[0] = (frame_coord[0] + 0.5f) / frame_dimensions[0]; + result[1] = (frame_coord[1] + 0.5f) / frame_dimensions[1]; +} + +static void libmv_frame_to_normalized_relative(const float frame_coord[2], + const float origin[2], + const int frame_dimensions[2], + float result[2]) +{ + result[0] = (frame_coord[0] - origin[0]) / frame_dimensions[0]; + result[1] = (frame_coord[1] - origin[1]) / frame_dimensions[1]; +} + +static void dna_marker_to_libmv_marker(/*const*/ MovieTrackingTrack *track, + /*const*/ MovieTrackingMarker *marker, + int clip, + int track_index, + int frame_width, + int frame_height, + bool backwards, + libmv_Marker *libmv_marker) +{ + const int frame_dimensions[2] = {frame_width, frame_height}; + int i; + libmv_marker->clip = clip; + libmv_marker->frame = marker->framenr; + libmv_marker->track = track_index; + + normalized_to_libmv_frame(marker->pos, + frame_dimensions, + libmv_marker->center); + for (i = 0; i < 4; ++i) { + normalized_relative_to_libmv_frame(marker->pattern_corners[i], + marker->pos, + frame_dimensions, + libmv_marker->patch[i]); + } + + normalized_relative_to_libmv_frame(marker->search_min, + marker->pos, + frame_dimensions, + libmv_marker->search_region_min); + + normalized_relative_to_libmv_frame(marker->search_max, + marker->pos, + frame_dimensions, + libmv_marker->search_region_max); + + /* TODO(sergey): All the markers does have 1.0 weight. */ + libmv_marker->weight = 1.0f; + + if (marker->flag & MARKER_TRACKED) { + libmv_marker->source = LIBMV_MARKER_SOURCE_TRACKED; + } + else { + libmv_marker->source = LIBMV_MARKER_SOURCE_MANUAL; + } + libmv_marker->status = LIBMV_MARKER_STATUS_UNKNOWN; + libmv_marker->model_type = LIBMV_MARKER_MODEL_TYPE_POINT; + libmv_marker->model_id = 0; + + /* TODO(sergey): We currently don't support reference marker from + * different clip. + */ + libmv_marker->reference_clip = clip; + + if (track->pattern_match == TRACK_MATCH_KEYFRAME) { + MovieTrackingMarker *keyframe_marker = + tracking_get_keyframed_marker(track, + marker->framenr, + backwards); + libmv_marker->reference_frame = keyframe_marker->framenr; + } + else { + libmv_marker->reference_frame = backwards ? + marker->framenr - 1 : + marker->framenr; + } +} + +static void libmv_marker_to_dna_marker(libmv_Marker *libmv_marker, + int frame_width, + int frame_height, + MovieTrackingMarker *marker) +{ + const int frame_dimensions[2] = {frame_width, frame_height}; + int i; + marker->framenr = libmv_marker->frame; + + libmv_frame_to_normalized(libmv_marker->center, + frame_dimensions, + marker->pos); + for (i = 0; i < 4; ++i) { + libmv_frame_to_normalized_relative(libmv_marker->patch[i], + libmv_marker->center, + frame_dimensions, + marker->pattern_corners[i]); + } + + libmv_frame_to_normalized_relative(libmv_marker->search_region_min, + libmv_marker->center, + frame_dimensions, + marker->search_min); + + libmv_frame_to_normalized_relative(libmv_marker->search_region_max, + libmv_marker->center, + frame_dimensions, + marker->search_max); + + marker->flag = 0; + if (libmv_marker->source == LIBMV_MARKER_SOURCE_TRACKED) { + marker->flag |= MARKER_TRACKED; + } + else { + marker->flag &= ~MARKER_TRACKED; + } +} + +static bool check_track_trackable(MovieClip *clip, + MovieTrackingTrack *track, + MovieClipUser *user) +{ + if (TRACK_SELECTED(track) && + (track->flag & (TRACK_LOCKED | TRACK_HIDDEN)) == 0) + { + MovieTrackingMarker *marker; + int frame; + frame = BKE_movieclip_remap_scene_to_clip_frame(clip, user->framenr); + marker = BKE_tracking_marker_get(track, frame); + return (marker->flag & MARKER_DISABLED) == 0; + } + return false; +} + +/* Returns false if marker crossed margin area from frame bounds. */ +static bool tracking_check_marker_margin(libmv_Marker *libmv_marker, + int margin, + int frame_width, + int frame_height) +{ + float patch_min[2], patch_max[2]; + float margin_left, margin_top, margin_right, margin_bottom; + + INIT_MINMAX2(patch_min, patch_max); + minmax_v2v2_v2(patch_min, patch_max, libmv_marker->patch[0]); + minmax_v2v2_v2(patch_min, patch_max, libmv_marker->patch[1]); + minmax_v2v2_v2(patch_min, patch_max, libmv_marker->patch[2]); + minmax_v2v2_v2(patch_min, patch_max, libmv_marker->patch[3]); + + margin_left = max_ff(libmv_marker->center[0] - patch_min[0], margin); + margin_top = max_ff(patch_max[1] - libmv_marker->center[1], margin); + margin_right = max_ff(patch_max[0] - libmv_marker->center[0], margin); + margin_bottom = max_ff(libmv_marker->center[1] - patch_min[1], margin); + + if (libmv_marker->center[0] < margin_left || + libmv_marker->center[0] > frame_width - margin_right || + libmv_marker->center[1] < margin_bottom || + libmv_marker->center[1] > frame_height - margin_top) + { + return false; + } + + return true; +} + +AutoTrackContext *BKE_autotrack_context_new(MovieClip *clip, + MovieClipUser *user, + const bool backwards, + const bool sequence) +{ + AutoTrackContext *context = MEM_callocN(sizeof(AutoTrackContext), + "autotrack context"); + MovieTracking *tracking = &clip->tracking; + MovieTrackingTrack *track; + ListBase *tracksbase = BKE_tracking_get_active_tracks(tracking); + int i, track_index, frame_width, frame_height; + + BKE_movieclip_get_size(clip, user, &frame_width, &frame_height); + + /* TODO(sergey): Currently using only a single clip. */ + context->clips[0] = clip; + context->num_clips = 1; + + context->user = *user; + context->user.render_size = MCLIP_PROXY_RENDER_SIZE_FULL; + context->user.render_flag = 0; + context->frame_width = frame_width; + context->frame_height = frame_height; + context->backwards = backwards; + context->sequence = sequence; + context->first_frame = user->framenr; + context->sync_frame = user->framenr; + context->first_sync = true; + + BLI_spin_init(&context->spin_lock); + + context->image_accessor = + tracking_image_accessor_new(context->clips, 1, user->framenr); + context->autotrack = + libmv_autoTrackNew(context->image_accessor->libmv_accessor); + + /* Fill in Autotrack with all markers we know. */ + track_index = 0; + for (track = tracksbase->first; + track; + track = track->next) + { + if (check_track_trackable(clip, track, user)) { + context->num_tracks++; + } + + for (i = 0; i < track->markersnr; ++i) { + MovieTrackingMarker *marker = marker = track->markers + i; + if ((marker->flag & MARKER_DISABLED) == 0) { + libmv_Marker libmv_marker; + dna_marker_to_libmv_marker(track, + marker, + 0, + track_index, + frame_width, + frame_height, + backwards, + &libmv_marker); + libmv_autoTrackAddMarker(context->autotrack, + &libmv_marker); + } + } + track_index++; + } + + /* Create per-track tracking options. */ + context->options = + MEM_callocN(sizeof(AutoTrackOptions) * context->num_tracks, + "auto track options"); + i = track_index = 0; + for (track = tracksbase->first; + track; + track = track->next) + { + if (check_track_trackable(clip, track, user)) { + AutoTrackOptions *options = &context->options[i++]; + /* TODO(sergey): Single clip only for now. */ + options->clip_index = 0; + options->track_index = track_index; + options->track = track; + tracking_configure_tracker(track, + NULL, + &options->track_region_options); + options->use_keyframe_match = + track->pattern_match == TRACK_MATCH_KEYFRAME; + } + ++track_index; + } + + return context; +} + +bool BKE_autotrack_context_step(AutoTrackContext *context) +{ + int frame_delta = context->backwards ? -1 : 1; + bool ok = false; + int track; + +#pragma omp parallel for if(context->num_tracks > 1) + for (track = 0; track < context->num_tracks; ++track) { + AutoTrackOptions *options = &context->options[track]; + libmv_Marker libmv_current_marker, + libmv_reference_marker, + libmv_tracked_marker; + libmv_TrackRegionResult libmv_result; + int frame = BKE_movieclip_remap_scene_to_clip_frame( + context->clips[options->clip_index], + context->user.framenr); + + if (libmv_autoTrackGetMarker(context->autotrack, + options->clip_index, + frame, + options->track_index, + &libmv_current_marker)) + { + if (!tracking_check_marker_margin(&libmv_current_marker, + options->track->margin, + context->frame_width, + context->frame_height)) + { + continue; + } + + libmv_tracked_marker = libmv_current_marker; + libmv_tracked_marker.frame = frame + frame_delta; + + if (options->use_keyframe_match) { + libmv_tracked_marker.reference_frame = + libmv_current_marker.reference_frame; + libmv_autoTrackGetMarker(context->autotrack, + options->clip_index, + libmv_tracked_marker.reference_frame, + options->track_index, + &libmv_reference_marker); + } + else { + libmv_tracked_marker.reference_frame = frame; + libmv_reference_marker = libmv_current_marker; + } + + if (libmv_autoTrackMarker(context->autotrack, + &options->track_region_options, + &libmv_tracked_marker, + &libmv_result)) + { + BLI_spin_lock(&context->spin_lock); + libmv_autoTrackAddMarker(context->autotrack, + &libmv_tracked_marker); + BLI_spin_unlock(&context->spin_lock); + } + else { + options->is_failed = true; + options->failed_frame = frame; + } + ok = true; + } + } + + BLI_spin_lock(&context->spin_lock); + context->user.framenr += frame_delta; + BLI_spin_unlock(&context->spin_lock); + + return ok; +} + +void BKE_autotrack_context_sync(AutoTrackContext *context) +{ + int newframe = context->user.framenr, + frame_delta = context->backwards ? -1 : 1; + int clip, frame; + + BLI_spin_lock(&context->spin_lock); + for (frame = context->sync_frame; + frame != (context->backwards ? newframe - 1 : newframe + 1); + frame += frame_delta) + { + MovieTrackingMarker marker; + libmv_Marker libmv_marker; + int clip = 0; + int track; + for (track = 0; track < context->num_tracks; ++track) { + AutoTrackOptions *options = &context->options[track]; + int track_frame = BKE_movieclip_remap_scene_to_clip_frame( + context->clips[options->clip_index], frame); + if (options->is_failed) { + if (options->failed_frame == track_frame) { + MovieTrackingMarker *prev_marker = + BKE_tracking_marker_get_exact( + options->track, + frame); + if (prev_marker) { + marker = *prev_marker; + marker.framenr = context->backwards ? + track_frame - 1 : + track_frame + 1; + marker.flag |= MARKER_DISABLED; + BKE_tracking_marker_insert(options->track, &marker); + } + } + continue; + } + if (libmv_autoTrackGetMarker(context->autotrack, + clip, + track_frame, + options->track_index, + &libmv_marker)) + { + libmv_marker_to_dna_marker(&libmv_marker, + context->frame_width, + context->frame_height, + &marker); + if (context->first_sync && frame == context->sync_frame) { + tracking_marker_insert_disabled(options->track, + &marker, + !context->backwards, + false); + } + BKE_tracking_marker_insert(options->track, &marker); + tracking_marker_insert_disabled(options->track, + &marker, + context->backwards, + false); + } + } + } + BLI_spin_unlock(&context->spin_lock); + + for (clip = 0; clip < context->num_clips; ++clip) { + MovieTracking *tracking = &context->clips[clip]->tracking; + BKE_tracking_dopesheet_tag_update(tracking); + } + + context->sync_frame = newframe; + context->first_sync = false; +} + +void BKE_autotrack_context_sync_user(AutoTrackContext *context, + MovieClipUser *user) +{ + user->framenr = context->sync_frame; +} + +void BKE_autotrack_context_finish(AutoTrackContext *context) +{ + int clip_index; + + for (clip_index = 0; clip_index < context->num_clips; ++clip_index) { + MovieClip *clip = context->clips[clip_index]; + ListBase *plane_tracks_base = + BKE_tracking_get_active_plane_tracks(&clip->tracking); + MovieTrackingPlaneTrack *plane_track; + + for (plane_track = plane_tracks_base->first; + plane_track; + plane_track = plane_track->next) + { + if ((plane_track->flag & PLANE_TRACK_AUTOKEY) == 0) { + int track; + for (track = 0; track < context->num_tracks; ++track) { + MovieTrackingTrack *old_track; + bool do_update = false; + int j; + + old_track = context->options[track].track; + for (j = 0; j < plane_track->point_tracksnr; j++) { + if (plane_track->point_tracks[j] == old_track) { + do_update = true; + break; + } + } + + if (do_update) { + BKE_tracking_track_plane_from_existing_motion( + plane_track, + context->first_frame); + break; + } + } + } + } + } +} + +void BKE_autotrack_context_free(AutoTrackContext *context) +{ + libmv_autoTrackDestroy(context->autotrack); + tracking_image_accessor_destroy(context->image_accessor); + MEM_freeN(context->options); + BLI_spin_end(&context->spin_lock); + MEM_freeN(context); +} diff --git a/source/blender/blenkernel/intern/tracking_region_tracker.c b/source/blender/blenkernel/intern/tracking_region_tracker.c index c41106f37cb..09c95ae4321 100644 --- a/source/blender/blenkernel/intern/tracking_region_tracker.c +++ b/source/blender/blenkernel/intern/tracking_region_tracker.c @@ -51,170 +51,6 @@ #include "libmv-capi.h" #include "tracking_private.h" -typedef struct TrackContext { - /* the reference marker and cutout search area */ - MovieTrackingMarker reference_marker; - - /* keyframed patch. This is the search area */ - float *search_area; - int search_area_height; - int search_area_width; - int framenr; - - float *mask; -} TrackContext; - -typedef struct MovieTrackingContext { - MovieClipUser user; - MovieClip *clip; - int clip_flag; - - int frames, first_frame; - bool first_time; - - MovieTrackingSettings settings; - TracksMap *tracks_map; - - bool backwards, sequence; - int sync_frame; -} MovieTrackingContext; - -static void track_context_free(void *customdata) -{ - TrackContext *track_context = (TrackContext *)customdata; - - if (track_context->search_area) - MEM_freeN(track_context->search_area); - - if (track_context->mask) - MEM_freeN(track_context->mask); -} - -/* Create context for motion 2D tracking, copies all data needed - * for thread-safe tracking, allowing clip modifications during - * tracking. - */ -MovieTrackingContext *BKE_tracking_context_new(MovieClip *clip, MovieClipUser *user, - const bool backwards, const bool sequence) -{ - MovieTrackingContext *context = MEM_callocN(sizeof(MovieTrackingContext), "trackingContext"); - MovieTracking *tracking = &clip->tracking; - MovieTrackingSettings *settings = &tracking->settings; - ListBase *tracksbase = BKE_tracking_get_active_tracks(tracking); - MovieTrackingTrack *track; - MovieTrackingObject *object = BKE_tracking_object_get_active(tracking); - int num_tracks = 0; - - context->clip = clip; - context->settings = *settings; - context->backwards = backwards; - context->sync_frame = user->framenr; - context->first_time = true; - context->first_frame = user->framenr; - context->sequence = sequence; - - /* count */ - track = tracksbase->first; - while (track) { - if (TRACK_SELECTED(track) && (track->flag & (TRACK_LOCKED | TRACK_HIDDEN)) == 0) { - int framenr = BKE_movieclip_remap_scene_to_clip_frame(clip, user->framenr); - MovieTrackingMarker *marker = BKE_tracking_marker_get(track, framenr); - - if ((marker->flag & MARKER_DISABLED) == 0) - num_tracks++; - } - - track = track->next; - } - - /* create tracking contextx for all tracks which would be tracked */ - if (num_tracks) { - int width, height; - - context->tracks_map = tracks_map_new(object->name, object->flag & TRACKING_OBJECT_CAMERA, - num_tracks, sizeof(TrackContext)); - - BKE_movieclip_get_size(clip, user, &width, &height); - - /* create tracking data */ - track = tracksbase->first; - while (track) { - if (TRACK_SELECTED(track) && (track->flag & (TRACK_HIDDEN | TRACK_LOCKED)) == 0) { - int framenr = BKE_movieclip_remap_scene_to_clip_frame(clip, user->framenr); - MovieTrackingMarker *marker = BKE_tracking_marker_get(track, framenr); - - if ((marker->flag & MARKER_DISABLED) == 0) { - TrackContext track_context; - memset(&track_context, 0, sizeof(TrackContext)); - tracks_map_insert(context->tracks_map, track, &track_context); - } - } - - track = track->next; - } - } - - /* store needed clip flags passing to get_buffer functions - * - MCLIP_USE_PROXY is needed to because timecode affects on movie clip - * only in case Proxy/Timecode flag is set, so store this flag to use - * timecodes properly but reset render size to SIZE_FULL so correct resolution - * would be used for images - * - MCLIP_USE_PROXY_CUSTOM_DIR is needed because proxy/timecode files might - * be stored in a different location - * ignore all the rest possible flags for now - */ - context->clip_flag = clip->flag & MCLIP_TIMECODE_FLAGS; - - context->user = *user; - context->user.render_size = MCLIP_PROXY_RENDER_SIZE_FULL; - context->user.render_flag = 0; - - if (!sequence) - BLI_begin_threaded_malloc(); - - return context; -} - -/* Free context used for tracking. */ -void BKE_tracking_context_free(MovieTrackingContext *context) -{ - if (!context->sequence) - BLI_end_threaded_malloc(); - - tracks_map_free(context->tracks_map, track_context_free); - - MEM_freeN(context); -} - -/* Synchronize tracks between clip editor and tracking context, - * by merging them together so all new created tracks and tracked - * ones presents in the movie clip. - */ -void BKE_tracking_context_sync(MovieTrackingContext *context) -{ - MovieTracking *tracking = &context->clip->tracking; - int newframe; - - tracks_map_merge(context->tracks_map, tracking); - - if (context->backwards) - newframe = context->user.framenr + 1; - else - newframe = context->user.framenr - 1; - - context->sync_frame = newframe; - - BKE_tracking_dopesheet_tag_update(tracking); -} - -/* Synchronize clip user's frame number with a frame number from tracking context, - * used to update current frame displayed in the clip editor while tracking. - */ -void BKE_tracking_context_sync_user(const MovieTrackingContext *context, MovieClipUser *user) -{ - user->framenr = context->sync_frame; -} - /* **** utility functions for tracking **** */ /* convert from float and byte RGBA to grayscale. Supports different coefficients for RGB. */ @@ -296,51 +132,6 @@ static ImBuf *tracking_context_get_frame_ibuf(MovieClip *clip, MovieClipUser *us return ibuf; } -/* Get previous keyframed marker. */ -static MovieTrackingMarker *tracking_context_get_keyframed_marker(MovieTrackingTrack *track, - int curfra, bool backwards) -{ - MovieTrackingMarker *marker_keyed = NULL; - MovieTrackingMarker *marker_keyed_fallback = NULL; - int a = BKE_tracking_marker_get(track, curfra) - track->markers; - - while (a >= 0 && a < track->markersnr) { - int next = backwards ? a + 1 : a - 1; - bool is_keyframed = false; - MovieTrackingMarker *cur_marker = &track->markers[a]; - MovieTrackingMarker *next_marker = NULL; - - if (next >= 0 && next < track->markersnr) - next_marker = &track->markers[next]; - - if ((cur_marker->flag & MARKER_DISABLED) == 0) { - /* If it'll happen so we didn't find a real keyframe marker, - * fallback to the first marker in current tracked segment - * as a keyframe. - */ - if (next_marker && next_marker->flag & MARKER_DISABLED) { - if (marker_keyed_fallback == NULL) - marker_keyed_fallback = cur_marker; - } - - is_keyframed |= (cur_marker->flag & MARKER_TRACKED) == 0; - } - - if (is_keyframed) { - marker_keyed = cur_marker; - - break; - } - - a = next; - } - - if (marker_keyed == NULL) - marker_keyed = marker_keyed_fallback; - - return marker_keyed; -} - /* Get image buffer for previous marker's keyframe. */ static ImBuf *tracking_context_get_keyframed_ibuf(MovieClip *clip, MovieClipUser *user, int clip_flag, MovieTrackingTrack *track, int curfra, bool backwards, @@ -349,7 +140,7 @@ static ImBuf *tracking_context_get_keyframed_ibuf(MovieClip *clip, MovieClipUser MovieTrackingMarker *marker_keyed; int keyed_framenr; - marker_keyed = tracking_context_get_keyframed_marker(track, curfra, backwards); + marker_keyed = tracking_get_keyframed_marker(track, curfra, backwards); if (marker_keyed == NULL) { return NULL; } @@ -381,50 +172,9 @@ static ImBuf *tracking_context_get_reference_ibuf(MovieClip *clip, MovieClipUser return ibuf; } -/* Update track's reference patch (patch from which track is tracking from) - * - * Returns false if reference image buffer failed to load. - */ -static bool track_context_update_reference(MovieTrackingContext *context, TrackContext *track_context, - MovieTrackingTrack *track, MovieTrackingMarker *marker, int curfra, - int frame_width, int frame_height) -{ - MovieTrackingMarker *reference_marker = NULL; - ImBuf *reference_ibuf = NULL; - int width, height; - - /* calculate patch for keyframed position */ - reference_ibuf = tracking_context_get_reference_ibuf(context->clip, &context->user, context->clip_flag, - track, curfra, context->backwards, &reference_marker); - - if (!reference_ibuf) - return false; - - track_context->reference_marker = *reference_marker; - - if (track_context->search_area) { - MEM_freeN(track_context->search_area); - } - - track_context->search_area = track_get_search_floatbuf(reference_ibuf, track, reference_marker, &width, &height); - track_context->search_area_height = height; - track_context->search_area_width = width; - - if ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_MASK) != 0) { - if (track_context->mask) - MEM_freeN(track_context->mask); - - track_context->mask = BKE_tracking_track_get_mask(frame_width, frame_height, track, marker); - } - - IMB_freeImBuf(reference_ibuf); - - return true; -} - /* Fill in libmv tracker options structure with settings need to be used to perform track. */ -static void tracking_configure_tracker(const MovieTrackingTrack *track, float *mask, - libmv_TrackRegionOptions *options) +void tracking_configure_tracker(const MovieTrackingTrack *track, float *mask, + libmv_TrackRegionOptions *options) { options->motion_model = track->motion_model; @@ -442,102 +192,6 @@ static void tracking_configure_tracker(const MovieTrackingTrack *track, float *m options->image1_mask = NULL; } -/* returns false if marker crossed margin area from frame bounds */ -static bool tracking_check_marker_margin(MovieTrackingTrack *track, MovieTrackingMarker *marker, - int frame_width, int frame_height) -{ - float pat_min[2], pat_max[2]; - float margin_left, margin_top, margin_right, margin_bottom; - float normalized_track_margin[2]; - - /* margin from frame boundaries */ - BKE_tracking_marker_pattern_minmax(marker, pat_min, pat_max); - - normalized_track_margin[0] = (float)track->margin / frame_width; - normalized_track_margin[1] = (float)track->margin / frame_height; - - margin_left = max_ff(-pat_min[0], normalized_track_margin[0]); - margin_top = max_ff( pat_max[1], normalized_track_margin[1]); - margin_right = max_ff( pat_max[0], normalized_track_margin[0]); - margin_bottom = max_ff(-pat_min[1], normalized_track_margin[1]); - - /* do not track markers which are too close to boundary */ - if (marker->pos[0] < margin_left || marker->pos[0] > 1.0f - margin_right || - marker->pos[1] < margin_bottom || marker->pos[1] > 1.0f - margin_top) - { - return false; - } - - return true; -} - -/* Scale search area of marker based on scale changes of pattern area, - * - * TODO(sergey): currently based on pattern bounding box scale change, - * smarter approach here is welcome. - */ -static void tracking_scale_marker_search(const MovieTrackingMarker *old_marker, MovieTrackingMarker *new_marker) -{ - float old_pat_min[2], old_pat_max[2]; - float new_pat_min[2], new_pat_max[2]; - float scale_x, scale_y; - - BKE_tracking_marker_pattern_minmax(old_marker, old_pat_min, old_pat_max); - BKE_tracking_marker_pattern_minmax(new_marker, new_pat_min, new_pat_max); - - scale_x = (new_pat_max[0] - new_pat_min[0]) / (old_pat_max[0] - old_pat_min[0]); - scale_y = (new_pat_max[1] - new_pat_min[1]) / (old_pat_max[1] - old_pat_min[1]); - - new_marker->search_min[0] *= scale_x; - new_marker->search_min[1] *= scale_y; - - new_marker->search_max[0] *= scale_x; - new_marker->search_max[1] *= scale_y; -} - -/* Insert new marker which was tracked from old_marker to a new image, - * will also ensure tracked segment is surrounded by disabled markers. - */ -static void tracking_insert_new_marker(MovieTrackingContext *context, MovieTrackingTrack *track, - const MovieTrackingMarker *old_marker, int curfra, bool tracked, - int frame_width, int frame_height, - const double dst_pixel_x[5], const double dst_pixel_y[5]) -{ - MovieTrackingMarker new_marker; - int frame_delta = context->backwards ? -1 : 1; - int nextfra = curfra + frame_delta; - - new_marker = *old_marker; - - if (tracked) { - tracking_set_marker_coords_from_tracking(frame_width, frame_height, &new_marker, dst_pixel_x, dst_pixel_y); - new_marker.flag |= MARKER_TRACKED; - new_marker.framenr = nextfra; - - tracking_scale_marker_search(old_marker, &new_marker); - - if (context->first_time) { - /* check if there's no keyframe/tracked markers before tracking marker. - * if so -- create disabled marker before currently tracking "segment" - */ - - tracking_marker_insert_disabled(track, old_marker, !context->backwards, false); - } - - /* insert currently tracked marker */ - BKE_tracking_marker_insert(track, &new_marker); - - /* make currently tracked segment be finished with disabled marker */ - tracking_marker_insert_disabled(track, &new_marker, context->backwards, false); - } - else { - new_marker.framenr = nextfra; - new_marker.flag |= MARKER_DISABLED; - - BKE_tracking_marker_insert(track, &new_marker); - } -} - /* Peform tracking from a reference_marker to destination_ibuf. * Uses marker as an initial position guess. * @@ -601,130 +255,6 @@ static bool configure_and_run_tracker(ImBuf *destination_ibuf, MovieTrackingTrac return tracked; } -/* Track all the tracks from context one more frame, - * returns FALSe if nothing was tracked. - */ -bool BKE_tracking_context_step(MovieTrackingContext *context) -{ - ImBuf *destination_ibuf; - int frame_delta = context->backwards ? -1 : 1; - int curfra = BKE_movieclip_remap_scene_to_clip_frame(context->clip, context->user.framenr); - int a, map_size; - bool ok = false; - - int frame_width, frame_height; - - map_size = tracks_map_get_size(context->tracks_map); - - /* Nothing to track, avoid unneeded frames reading to save time and memory. */ - if (!map_size) - return false; - - /* Get an image buffer for frame we're tracking to. */ - context->user.framenr += frame_delta; - - destination_ibuf = BKE_movieclip_get_ibuf_flag(context->clip, &context->user, - context->clip_flag, MOVIECLIP_CACHE_SKIP); - if (!destination_ibuf) - return false; - - frame_width = destination_ibuf->x; - frame_height = destination_ibuf->y; - -#pragma omp parallel for private(a) shared(destination_ibuf, ok) if (map_size > 1) - for (a = 0; a < map_size; a++) { - TrackContext *track_context = NULL; - MovieTrackingTrack *track; - MovieTrackingMarker *marker; - - tracks_map_get_indexed_element(context->tracks_map, a, &track, (void **)&track_context); - - marker = BKE_tracking_marker_get_exact(track, curfra); - - if (marker && (marker->flag & MARKER_DISABLED) == 0) { - bool tracked = false, need_readjust; - double dst_pixel_x[5], dst_pixel_y[5]; - - if (track->pattern_match == TRACK_MATCH_KEYFRAME) - need_readjust = context->first_time; - else - need_readjust = true; - - /* do not track markers which are too close to boundary */ - if (tracking_check_marker_margin(track, marker, frame_width, frame_height)) { - if (need_readjust) { - if (track_context_update_reference(context, track_context, track, marker, - curfra, frame_width, frame_height) == false) - { - /* happens when reference frame fails to be loaded */ - continue; - } - } - - tracked = configure_and_run_tracker(destination_ibuf, track, - &track_context->reference_marker, marker, - track_context->search_area, - track_context->search_area_width, - track_context->search_area_height, - track_context->mask, - dst_pixel_x, dst_pixel_y); - } - - BLI_spin_lock(&context->tracks_map->spin_lock); - tracking_insert_new_marker(context, track, marker, curfra, tracked, - frame_width, frame_height, dst_pixel_x, dst_pixel_y); - BLI_spin_unlock(&context->tracks_map->spin_lock); - - ok = true; - } - } - - IMB_freeImBuf(destination_ibuf); - - context->first_time = false; - context->frames++; - - return ok; -} - -void BKE_tracking_context_finish(MovieTrackingContext *context) -{ - MovieClip *clip = context->clip; - ListBase *plane_tracks_base = BKE_tracking_get_active_plane_tracks(&clip->tracking); - MovieTrackingPlaneTrack *plane_track; - int map_size = tracks_map_get_size(context->tracks_map); - - for (plane_track = plane_tracks_base->first; - plane_track; - plane_track = plane_track->next) - { - if ((plane_track->flag & PLANE_TRACK_AUTOKEY) == 0) { - int i; - for (i = 0; i < map_size; i++) { - TrackContext *track_context = NULL; - MovieTrackingTrack *track, *old_track; - bool do_update = false; - int j; - - tracks_map_get_indexed_element(context->tracks_map, i, &track, (void **)&track_context); - - old_track = BLI_ghash_lookup(context->tracks_map->hash, track); - for (j = 0; j < plane_track->point_tracksnr; j++) { - if (plane_track->point_tracks[j] == old_track) { - do_update = true; - break; - } - } - - if (do_update) { - BKE_tracking_track_plane_from_existing_motion(plane_track, context->first_frame); - break; - } - } - } - } -} - static bool refine_marker_reference_frame_get(MovieTrackingTrack *track, MovieTrackingMarker *marker, bool backwards, diff --git a/source/blender/blenkernel/intern/tracking_util.c b/source/blender/blenkernel/intern/tracking_util.c index 0a8293630c7..250b575aa14 100644 --- a/source/blender/blenkernel/intern/tracking_util.c +++ b/source/blender/blenkernel/intern/tracking_util.c @@ -47,8 +47,13 @@ #include "BLF_translation.h" +#include "BKE_movieclip.h" #include "BKE_tracking.h" +#include "IMB_imbuf_types.h" +#include "IMB_imbuf.h" +#include "IMB_moviecache.h" + #include "tracking_private.h" #include "libmv-capi.h" @@ -390,8 +395,7 @@ void tracking_marker_insert_disabled(MovieTrackingTrack *track, const MovieTrack } -/* Fill in Libmv C-API camera intrinsics options from tracking structure. - */ +/* Fill in Libmv C-API camera intrinsics options from tracking structure. */ void tracking_cameraIntrinscisOptionsFromTracking(MovieTracking *tracking, int calibration_width, int calibration_height, libmv_CameraIntrinsicsOptions *camera_intrinsics_options) @@ -453,3 +457,439 @@ void tracking_trackingCameraFromIntrinscisOptions(MovieTracking *tracking, BLI_assert(!"Unknown distortion model"); } } + +/* Get previous keyframed marker. */ +MovieTrackingMarker *tracking_get_keyframed_marker(MovieTrackingTrack *track, + int current_frame, + bool backwards) +{ + MovieTrackingMarker *marker_keyed = NULL; + MovieTrackingMarker *marker_keyed_fallback = NULL; + int a = BKE_tracking_marker_get(track, current_frame) - track->markers; + + while (a >= 0 && a < track->markersnr) { + int next = backwards ? a + 1 : a - 1; + bool is_keyframed = false; + MovieTrackingMarker *cur_marker = &track->markers[a]; + MovieTrackingMarker *next_marker = NULL; + + if (next >= 0 && next < track->markersnr) + next_marker = &track->markers[next]; + + if ((cur_marker->flag & MARKER_DISABLED) == 0) { + /* If it'll happen so we didn't find a real keyframe marker, + * fallback to the first marker in current tracked segment + * as a keyframe. + */ + if (next_marker && next_marker->flag & MARKER_DISABLED) { + if (marker_keyed_fallback == NULL) + marker_keyed_fallback = cur_marker; + } + + is_keyframed |= (cur_marker->flag & MARKER_TRACKED) == 0; + } + + if (is_keyframed) { + marker_keyed = cur_marker; + + break; + } + + a = next; + } + + if (marker_keyed == NULL) + marker_keyed = marker_keyed_fallback; + + return marker_keyed; +} + +/*********************** Frame accessr *************************/ + +typedef struct AccessCacheKey { + int clip_index; + int frame; + int downscale; + libmv_InputMode input_mode; + int64_t transform_key; +} AccessCacheKey; + +static unsigned int accesscache_hashhash(const void *key_v) +{ + const AccessCacheKey *key = (const AccessCacheKey *) key_v; + /* TODP(sergey): Need better hasing here for faster frame access. */ + return key->clip_index << 16 | key->frame; +} + +static bool accesscache_hashcmp(const void *a_v, const void *b_v) +{ + const AccessCacheKey *a = (const AccessCacheKey *) a_v; + const AccessCacheKey *b = (const AccessCacheKey *) b_v; + +#define COMPARE_FIELD(field) + { \ + if (a->clip_index != b->clip_index) { \ + return false; \ + } \ + } (void) 0 + + COMPARE_FIELD(clip_index); + COMPARE_FIELD(frame); + COMPARE_FIELD(downscale); + COMPARE_FIELD(input_mode); + COMPARE_FIELD(transform_key); + +#undef COMPARE_FIELD + + return true; +} + +static void accesscache_put(TrackingImageAccessor *accessor, + int clip_index, + int frame, + libmv_InputMode input_mode, + int downscale, + int64_t transform_key, + ImBuf *ibuf) +{ + AccessCacheKey key; + key.clip_index = clip_index; + key.frame = frame; + key.input_mode = input_mode; + key.downscale = downscale; + key.transform_key = transform_key; + IMB_moviecache_put(accessor->cache, &key, ibuf); +} + +static ImBuf *accesscache_get(TrackingImageAccessor *accessor, + int clip_index, + int frame, + libmv_InputMode input_mode, + int downscale, + int64_t transform_key) +{ + AccessCacheKey key; + key.clip_index = clip_index; + key.frame = frame; + key.input_mode = input_mode; + key.downscale = downscale; + key.transform_key = transform_key; + return IMB_moviecache_get(accessor->cache, &key); +} + +static ImBuf *accessor_get_preprocessed_ibuf(TrackingImageAccessor *accessor, + int clip_index, + int frame) +{ + MovieClip *clip; + MovieClipUser user; + ImBuf *ibuf; + int scene_frame; + + BLI_assert(clip_index < accessor->num_clips); + + clip = accessor->clips[clip_index]; + scene_frame = BKE_movieclip_remap_clip_to_scene_frame(clip, frame); + BKE_movieclip_user_set_frame(&user, scene_frame); + user.render_size = MCLIP_PROXY_RENDER_SIZE_FULL; + user.render_flag = 0; + ibuf = BKE_movieclip_get_ibuf(clip, &user); + + return ibuf; +} + +static ImBuf *make_grayscale_ibuf_copy(ImBuf *ibuf) +{ + ImBuf *grayscale = IMB_allocImBuf(ibuf->x, ibuf->y, 32, 0); + size_t size; + int i; + + BLI_assert(ibuf->channels == 3 || ibuf->channels == 4); + + /* TODO(sergey): Bummer, currently IMB API only allows to create 4 channels + * float buffer, so we do it manually here. + * + * Will generalize it later. + */ + size = (size_t)grayscale->x * (size_t)grayscale->y * sizeof(float); + grayscale->channels = 1; + if ((grayscale->rect_float = MEM_mapallocN(size, "tracking grayscale image"))) { + grayscale->mall |= IB_rectfloat; + grayscale->flags |= IB_rectfloat; + } + + for (i = 0; i < grayscale->x * grayscale->y; ++i) { + const float *pixel = ibuf->rect_float + ibuf->channels * i; + + grayscale->rect_float[i] = 0.2126f * pixel[0] + + 0.7152f * pixel[1] + + 0.0722f * pixel[2]; + } + + return grayscale; +} + +static void ibuf_to_float_image(const ImBuf *ibuf, libmv_FloatImage *float_image) +{ + BLI_assert(ibuf->rect_float != NULL); + float_image->buffer = ibuf->rect_float; + float_image->width = ibuf->x; + float_image->height = ibuf->y; + float_image->channels = ibuf->channels; +} + +static ImBuf *float_image_to_ibuf(libmv_FloatImage *float_image) +{ + ImBuf *ibuf = IMB_allocImBuf(float_image->width, float_image->height, 32, 0); + size_t size = (size_t)ibuf->x * (size_t)ibuf->y * + float_image->channels * sizeof(float); + ibuf->channels = float_image->channels; + if ((ibuf->rect_float = MEM_mapallocN(size, "tracking grayscale image"))) { + ibuf->mall |= IB_rectfloat; + ibuf->flags |= IB_rectfloat; + } + memcpy(ibuf->rect_float, float_image->buffer, size); + return ibuf; +} + +static ImBuf *accessor_get_ibuf(TrackingImageAccessor *accessor, + int clip_index, + int frame, + libmv_InputMode input_mode, + int downscale, + const libmv_Region *region, + const libmv_FrameTransform *transform) +{ + ImBuf *ibuf, *orig_ibuf, *final_ibuf; + int64_t transform_key = 0; + + if (transform != NULL) { + transform_key = libmv_frameAccessorgetTransformKey(transform); + } + + /* First try to get fully processed image from the cache. */ + ibuf = accesscache_get(accessor, + clip_index, + frame, + input_mode, + downscale, + transform_key); + if (ibuf != NULL) { + return ibuf; + } + + /* And now we do postprocessing of the original frame. */ + orig_ibuf = accessor_get_preprocessed_ibuf(accessor, clip_index, frame); + + if (orig_ibuf == NULL) { + return NULL; + } + + if (region != NULL) { + int width = region->max[0] - region->min[0], + height = region->max[1] - region->min[1]; + + /* If the requested region goes outside of the actual frame we still + * return the requested region size, but only fill it's partially with + * the data we can. + */ + int clamped_origin_x = max_ii((int)region->min[0], 0), + clamped_origin_y = max_ii((int)region->min[1], 0); + int dst_offset_x = clamped_origin_x - (int)region->min[0], + dst_offset_y = clamped_origin_y - (int)region->min[1]; + int clamped_width = width - dst_offset_x, + clamped_height = height - dst_offset_y; + clamped_width = min_ii(clamped_width, orig_ibuf->x - clamped_origin_x); + clamped_height = min_ii(clamped_height, orig_ibuf->y - clamped_origin_y); + + final_ibuf = IMB_allocImBuf(width, height, 32, IB_rectfloat); + + if (orig_ibuf->rect_float != NULL) { + IMB_rectcpy(final_ibuf, orig_ibuf, + dst_offset_x, dst_offset_y, + clamped_origin_x, clamped_origin_y, + clamped_width, clamped_height); + } + else { + int y; + /* TODO(sergey): We don't do any color space or alpha conversion + * here. Probably Libmv is better to work in the linear space, + * but keep sRGB space here for compatibility for now. + */ + for (y = 0; y < clamped_height; ++y) { + int x; + for (x = 0; x < clamped_width; ++x) { + int src_x = x + clamped_origin_x, + src_y = y + clamped_origin_y; + int dst_x = x + dst_offset_x, + dst_y = y + dst_offset_y; + int dst_index = (dst_y * width + dst_x) * 4, + src_index = (src_y * orig_ibuf->x + src_x) * 4; + rgba_uchar_to_float(final_ibuf->rect_float + dst_index, + (unsigned char *)orig_ibuf->rect + + src_index); + } + } + } + } + else { + /* Libmv only works with float images, + * + * This would likely make it so loads of float buffers are being stored + * in the cache which is nice on the one hand (faster re-use of the + * frames) but on the other hand it bumps the memory usage up. + */ + BLI_lock_thread(LOCK_MOVIECLIP); + IMB_float_from_rect(orig_ibuf); + BLI_unlock_thread(LOCK_MOVIECLIP); + final_ibuf = orig_ibuf; + } + + if (downscale > 0) { + if (final_ibuf == orig_ibuf) { + final_ibuf = IMB_dupImBuf(orig_ibuf); + } + IMB_scaleImBuf(final_ibuf, + ibuf->x / (1 << downscale), + ibuf->y / (1 << downscale)); + } + + if (input_mode == LIBMV_IMAGE_MODE_RGBA) { + BLI_assert(ibuf->channels == 3 || ibuf->channels == 4); + /* pass */ + } + else /* if (input_mode == LIBMV_IMAGE_MODE_MONO) */ { + ImBuf *grayscale_ibuf = make_grayscale_ibuf_copy(final_ibuf); + if (final_ibuf != orig_ibuf) { + /* We dereference original frame later. */ + IMB_freeImBuf(final_ibuf); + } + final_ibuf = grayscale_ibuf; + } + + if (transform != NULL) { + libmv_FloatImage input_image, output_image; + ibuf_to_float_image(final_ibuf, &input_image); + libmv_frameAccessorgetTransformRun(transform, + &input_image, + &output_image); + if (final_ibuf != orig_ibuf) { + IMB_freeImBuf(final_ibuf); + } + final_ibuf = float_image_to_ibuf(&output_image); + libmv_floatImageDestroy(&output_image); + } + + /* it's possible processing stil didn't happen at this point, + * but we really need a copy of the buffer to be transformed + * and to be put to the cache. + */ + if (final_ibuf == orig_ibuf) { + final_ibuf = IMB_dupImBuf(orig_ibuf); + } + + IMB_freeImBuf(orig_ibuf); + + /* We put postprocessed frame to the cache always for now, + * not the smartest thing in the world, but who cares at this point. + */ + + /* TODO(sergey): Disable cache for now, because we don't store region + * in the cache key and can't check whether cached version is usable for + * us or not. + * + * Need to think better about what to cache and when. + */ + if (false) { + accesscache_put(accessor, + clip_index, + frame, + input_mode, + downscale, + transform_key, + final_ibuf); + } + + return final_ibuf; +} + +static libmv_CacheKey accessor_get_image_callback( + struct libmv_FrameAccessorUserData *user_data, + int clip_index, + int frame, + libmv_InputMode input_mode, + int downscale, + const libmv_Region *region, + const libmv_FrameTransform *transform, + float **destination, + int *width, + int *height, + int *channels) +{ + TrackingImageAccessor *accessor = (TrackingImageAccessor *) user_data; + ImBuf *ibuf; + + BLI_assert(clip_index >= 0 && clip_index < accessor->num_clips); + + ibuf = accessor_get_ibuf(accessor, + clip_index, + frame, + input_mode, + downscale, + region, + transform); + + if (ibuf) { + *destination = ibuf->rect_float; + *width = ibuf->x; + *height = ibuf->y; + *channels = ibuf->channels; + } + else { + *destination = NULL; + *width = 0; + *height = 0; + *channels = 0; + } + + return ibuf; +} + +static void accessor_release_image_callback(libmv_CacheKey cache_key) +{ + ImBuf *ibuf = (ImBuf *) cache_key; + IMB_freeImBuf(ibuf); +} + +TrackingImageAccessor *tracking_image_accessor_new(MovieClip *clips[MAX_ACCESSOR_CLIP], + int num_clips, + int start_frame) +{ + TrackingImageAccessor *accessor = + MEM_callocN(sizeof(TrackingImageAccessor), "tracking image accessor"); + + BLI_assert(num_clips <= MAX_ACCESSOR_CLIP); + + accessor->cache = IMB_moviecache_create("frame access cache", + sizeof(AccessCacheKey), + accesscache_hashhash, + accesscache_hashcmp); + + memcpy(accessor->clips, clips, num_clips * sizeof(MovieClip*)); + accessor->num_clips = num_clips; + accessor->start_frame = start_frame; + + accessor->libmv_accessor = + libmv_FrameAccessorNew((libmv_FrameAccessorUserData *) accessor, + accessor_get_image_callback, + accessor_release_image_callback); + + return accessor; +} + +void tracking_image_accessor_destroy(TrackingImageAccessor *accessor) +{ + IMB_moviecache_free(accessor->cache); + libmv_FrameAccessorDestroy(accessor->libmv_accessor); + MEM_freeN(accessor); +} diff --git a/source/blender/blenkernel/tracking_private.h b/source/blender/blenkernel/tracking_private.h index 6efa1533e3e..591ee4d0d01 100644 --- a/source/blender/blenkernel/tracking_private.h +++ b/source/blender/blenkernel/tracking_private.h @@ -95,4 +95,32 @@ void tracking_cameraIntrinscisOptionsFromTracking(struct MovieTracking *tracking void tracking_trackingCameraFromIntrinscisOptions(struct MovieTracking *tracking, const struct libmv_CameraIntrinsicsOptions *camera_intrinsics_options); +struct libmv_TrackRegionOptions; + +void tracking_configure_tracker(const MovieTrackingTrack *track, float *mask, + struct libmv_TrackRegionOptions *options); + +struct MovieTrackingMarker *tracking_get_keyframed_marker( + struct MovieTrackingTrack *track, + int current_frame, + bool backwards); + +/*********************** Frame accessr *************************/ + +struct libmv_FrameAccessor; + +#define MAX_ACCESSOR_CLIP 64 +typedef struct TrackingImageAccessor { + struct MovieCache *cache; + struct MovieClip *clips[MAX_ACCESSOR_CLIP]; + int num_clips; + int start_frame; + struct libmv_FrameAccessor *libmv_accessor; +} TrackingImageAccessor; + +TrackingImageAccessor *tracking_image_accessor_new(MovieClip *clips[MAX_ACCESSOR_CLIP], + int num_clips, + int start_frame); +void tracking_image_accessor_destroy(TrackingImageAccessor *accessor); + #endif /* __TRACKING_PRIVATE_H__ */ diff --git a/source/blender/editors/space_clip/space_clip.c b/source/blender/editors/space_clip/space_clip.c index a797a60f74c..497c3dd2c96 100644 --- a/source/blender/editors/space_clip/space_clip.c +++ b/source/blender/editors/space_clip/space_clip.c @@ -1159,7 +1159,7 @@ static void clip_main_area_draw(const bContext *C, ARegion *ar) /* if tracking is in progress, we should synchronize framenr from clipuser * so latest tracked frame would be shown */ if (clip && clip->tracking_context) - BKE_tracking_context_sync_user(clip->tracking_context, &sc->user); + BKE_autotrack_context_sync_user(clip->tracking_context, &sc->user); if (sc->flag & SC_LOCK_SELECTION) { ImBuf *tmpibuf = NULL; diff --git a/source/blender/editors/space_clip/tracking_ops.c b/source/blender/editors/space_clip/tracking_ops.c index fb6b1a0033c..bdb57cca81e 100644 --- a/source/blender/editors/space_clip/tracking_ops.c +++ b/source/blender/editors/space_clip/tracking_ops.c @@ -1083,7 +1083,7 @@ void CLIP_OT_slide_marker(wmOperatorType *ot) /********************** track operator *********************/ typedef struct TrackMarkersJob { - struct MovieTrackingContext *context; /* tracking context */ + struct AutoTrackContext *context; /* tracking context */ int sfra, efra, lastfra; /* Start, end and recently tracked frames */ int backwards; /* Backwards tracking flag */ MovieClip *clip; /* Clip which is tracking */ @@ -1231,7 +1231,7 @@ static int track_markers_initjob(bContext *C, TrackMarkersJob *tmj, int backward tmj->delay /= 2; } - tmj->context = BKE_tracking_context_new(clip, &sc->user, backwards, 1); + tmj->context = BKE_autotrack_context_new(clip, &sc->user, backwards, 1); clip->tracking_context = tmj->context; @@ -1265,14 +1265,14 @@ static void track_markers_startjob(void *tmv, short *stop, short *do_update, flo double start_time = PIL_check_seconds_timer(), exec_time; - if (!BKE_tracking_context_step(tmj->context)) + if (!BKE_autotrack_context_step(tmj->context)) break; exec_time = PIL_check_seconds_timer() - start_time; if (tmj->delay > (float)exec_time) PIL_sleep_ms(tmj->delay - (float)exec_time); } - else if (!BKE_tracking_context_step(tmj->context)) + else if (!BKE_autotrack_context_step(tmj->context)) break; *do_update = true; @@ -1296,7 +1296,7 @@ static void track_markers_updatejob(void *tmv) { TrackMarkersJob *tmj = (TrackMarkersJob *)tmv; - BKE_tracking_context_sync(tmj->context); + BKE_autotrack_context_sync(tmj->context); } static void track_markers_endjob(void *tmv) @@ -1310,8 +1310,8 @@ static void track_markers_endjob(void *tmv) ED_update_for_newframe(tmj->main, tmj->scene, 0); } - BKE_tracking_context_sync(tmj->context); - BKE_tracking_context_finish(tmj->context); + BKE_autotrack_context_sync(tmj->context); + BKE_autotrack_context_finish(tmj->context); WM_main_add_notifier(NC_SCENE | ND_FRAME, tmj->scene); } @@ -1319,7 +1319,7 @@ static void track_markers_endjob(void *tmv) static void track_markers_freejob(void *tmv) { TrackMarkersJob *tmj = (TrackMarkersJob *)tmv; - BKE_tracking_context_free(tmj->context); + BKE_autotrack_context_free(tmj->context); MEM_freeN(tmj); } @@ -1328,7 +1328,7 @@ static int track_markers_exec(bContext *C, wmOperator *op) SpaceClip *sc; MovieClip *clip; Scene *scene = CTX_data_scene(C); - struct MovieTrackingContext *context; + struct AutoTrackContext *context; MovieClipUser *user, fake_user = {0}; int framenr, sfra, efra; const bool backwards = RNA_boolean_get(op->ptr, "backwards"); @@ -1388,10 +1388,10 @@ static int track_markers_exec(bContext *C, wmOperator *op) return OPERATOR_CANCELLED; /* do not disable tracks due to threshold when tracking frame-by-frame */ - context = BKE_tracking_context_new(clip, user, backwards, sequence); + context = BKE_autotrack_context_new(clip, user, backwards, sequence); while (framenr != efra) { - if (!BKE_tracking_context_step(context)) + if (!BKE_autotrack_context_step(context)) break; if (backwards) framenr--; @@ -1401,9 +1401,9 @@ static int track_markers_exec(bContext *C, wmOperator *op) break; } - BKE_tracking_context_sync(context); - BKE_tracking_context_finish(context); - BKE_tracking_context_free(context); + BKE_autotrack_context_sync(context); + BKE_autotrack_context_finish(context); + BKE_autotrack_context_free(context); /* update scene current frame to the lastes tracked frame */ scene->r.cfra = BKE_movieclip_remap_clip_to_scene_frame(clip, framenr); |