diff options
52 files changed, 5380 insertions, 190 deletions
diff --git a/intern/libmv/CMakeLists.txt b/intern/libmv/CMakeLists.txt index cd89f1d84b5..e277405cc8e 100644 --- a/intern/libmv/CMakeLists.txt +++ b/intern/libmv/CMakeLists.txt @@ -70,11 +70,18 @@ if(WITH_LIBMV) intern/image.cc intern/logging.cc intern/reconstruction.cc + intern/reconstructionN.cc intern/track_region.cc intern/tracks.cc intern/tracksN.cc libmv/autotrack/autotrack.cc + libmv/autotrack/bundle.cc + libmv/autotrack/intersect.cc + libmv/autotrack/keyframe_selection.cc + libmv/autotrack/pipeline.cc libmv/autotrack/predict_tracks.cc + libmv/autotrack/reconstruction.cc + libmv/autotrack/resect.cc libmv/autotrack/tracks.cc libmv/base/aligned_malloc.cc libmv/image/array_nd.cc @@ -119,18 +126,24 @@ if(WITH_LIBMV) intern/image.h intern/logging.h intern/reconstruction.h + intern/reconstructionN.h intern/track_region.h intern/tracks.h intern/tracksN.h libmv/autotrack/autotrack.h + libmv/autotrack/bundle.h libmv/autotrack/callbacks.h libmv/autotrack/frame_accessor.h + libmv/autotrack/intersect.h + libmv/autotrack/keyframe_selection.h libmv/autotrack/marker.h libmv/autotrack/model.h + libmv/autotrack/pipeline.h libmv/autotrack/predict_tracks.h libmv/autotrack/quad.h libmv/autotrack/reconstruction.h libmv/autotrack/region.h + libmv/autotrack/resect.h libmv/autotrack/tracks.h libmv/base/aligned_malloc.h libmv/base/id_generator.h diff --git a/intern/libmv/intern/reconstructionN.cc b/intern/libmv/intern/reconstructionN.cc new file mode 100644 index 00000000000..2c6a6d741d4 --- /dev/null +++ b/intern/libmv/intern/reconstructionN.cc @@ -0,0 +1,541 @@ +/* + * ***** BEGIN GPL LICENSE BLOCK ***** + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2011 Blender Foundation. + * All rights reserved. + * + * Contributor(s): Blender Foundation, + * Tianwei Shen + * + * ***** END GPL LICENSE BLOCK ***** + */ + +#include "intern/reconstructionN.h" +#include "intern/camera_intrinsics.h" +#include "intern/tracksN.h" +#include "intern/utildefines.h" + +#include "libmv/logging/logging.h" +#include "libmv/autotrack/autotrack.h" +#include "libmv/autotrack/bundle.h" +#include "libmv/autotrack/frame_accessor.h" +#include "libmv/autotrack/keyframe_selection.h" +#include "libmv/autotrack/marker.h" +#include "libmv/autotrack/model.h" +#include "libmv/autotrack/pipeline.h" +#include "libmv/autotrack/predict_tracks.h" +#include "libmv/autotrack/quad.h" +#include "libmv/autotrack/reconstruction.h" +#include "libmv/autotrack/region.h" +#include "libmv/autotrack/tracks.h" +#include "libmv/simple_pipeline/callbacks.h" + +using mv::Tracks; +using mv::Marker; +using mv::Reconstruction; + +using libmv::CameraIntrinsics; +using libmv::ProgressUpdateCallback; + +struct libmv_ReconstructionN { + mv::Reconstruction reconstruction; + + /* Used for per-track average error calculation after reconstruction */ + mv::Tracks tracks; + libmv::CameraIntrinsics *intrinsics; + double error; + bool is_valid; +}; + +namespace { +class MultiviewReconstructUpdateCallback : public ProgressUpdateCallback { +public: + MultiviewReconstructUpdateCallback( + multiview_reconstruct_progress_update_cb progress_update_callback, + void *callback_customdata) { + progress_update_callback_ = progress_update_callback; + callback_customdata_ = callback_customdata; + } + + void invoke(double progress, const char* message) { + if (progress_update_callback_) { + progress_update_callback_(callback_customdata_, progress, message); + } + } +protected: + multiview_reconstruct_progress_update_cb progress_update_callback_; + void* callback_customdata_; +}; + +void mv_getNormalizedTracks(const Tracks &tracks, + const CameraIntrinsics &camera_intrinsics, + Tracks *normalized_tracks) +{ + libmv::vector<Marker> markers = tracks.markers(); + for (int i = 0; i < markers.size(); ++i) { + Marker &marker = markers[i]; + + // act in a calibrated fashion + double normalized_x, normalized_y; + camera_intrinsics.InvertIntrinsics(marker.center[0], marker.center[1], + &normalized_x, &normalized_y); + marker.center[0] = normalized_x; + marker.center[1] = normalized_y; + + normalized_tracks->AddMarker(marker); + } +} + +// Each clip has a fix camera intrinsics, set frames of a clip to that fixed intrinsics +bool ReconstructionUpdateFixedIntrinsics(libmv_ReconstructionN **all_libmv_reconstruction, + Tracks *tracks, + Reconstruction *reconstruction) +{ + int clip_num = tracks->MaxClip() + 1; + for (int i = 0; i < clip_num; i++) { + CameraIntrinsics *camera_intrinsics = all_libmv_reconstruction[i]->intrinsics; + int cam_intrinsic_index = reconstruction->AddCameraIntrinsics(camera_intrinsics); + assert(cam_intrinsic_index == i); + } + reconstruction->InitIntrinsicsMapFixed(*tracks); + + return true; +} + +void libmv_solveRefineIntrinsics( + const Tracks &tracks, + const int refine_intrinsics, + const int bundle_constraints, + reconstruct_progress_update_cb progress_update_callback, + void* callback_customdata, + Reconstruction* reconstruction, + CameraIntrinsics* intrinsics) { + /* only a few combinations are supported but trust the caller/ */ + int bundle_intrinsics = 0; + + if (refine_intrinsics & LIBMV_REFINE_FOCAL_LENGTH) { + bundle_intrinsics |= mv::BUNDLE_FOCAL_LENGTH; + } + if (refine_intrinsics & LIBMV_REFINE_PRINCIPAL_POINT) { + bundle_intrinsics |= mv::BUNDLE_PRINCIPAL_POINT; + } + if (refine_intrinsics & LIBMV_REFINE_RADIAL_DISTORTION_K1) { + bundle_intrinsics |= mv::BUNDLE_RADIAL_K1; + } + if (refine_intrinsics & LIBMV_REFINE_RADIAL_DISTORTION_K2) { + bundle_intrinsics |= mv::BUNDLE_RADIAL_K2; + } + + progress_update_callback(callback_customdata, 1.0, "Refining solution"); + + mv::EuclideanBundleCommonIntrinsics(tracks, + bundle_intrinsics, + bundle_constraints, + reconstruction, + intrinsics); +} + +void finishMultiviewReconstruction( + const Tracks &tracks, + const CameraIntrinsics &camera_intrinsics, + libmv_ReconstructionN *libmv_reconstruction, + reconstruct_progress_update_cb progress_update_callback, + void *callback_customdata) { + Reconstruction &reconstruction = libmv_reconstruction->reconstruction; + + /* Reprojection error calculation. */ + progress_update_callback(callback_customdata, 1.0, "Finishing solution"); + libmv_reconstruction->tracks = tracks; + libmv_reconstruction->error = mv::EuclideanReprojectionError(tracks, + reconstruction, + camera_intrinsics); +} + +bool selectTwoClipKeyframesBasedOnGRICAndVariance( + const int clip_index, + Tracks& tracks, + Tracks& normalized_tracks, + CameraIntrinsics& camera_intrinsics, + int& keyframe1, + int& keyframe2) { + libmv::vector<int> keyframes; + + /* Get list of all keyframe candidates first. */ + mv::SelectClipKeyframesBasedOnGRICAndVariance(clip_index, + normalized_tracks, + camera_intrinsics, + keyframes); + + if (keyframes.size() < 2) { + LG << "Not enough keyframes detected by GRIC"; + return false; + } else if (keyframes.size() == 2) { + keyframe1 = keyframes[0]; + keyframe2 = keyframes[1]; + return true; + } + + /* Now choose two keyframes with minimal reprojection error after initial + * reconstruction choose keyframes with the least reprojection error after + * solving from two candidate keyframes. + * + * In fact, currently libmv returns single pair only, so this code will + * not actually run. But in the future this could change, so let's stay + * prepared. + */ + int previous_keyframe = keyframes[0]; + double best_error = std::numeric_limits<double>::max(); + for (int i = 1; i < keyframes.size(); i++) { + Reconstruction reconstruction; + int current_keyframe = keyframes[i]; + libmv::vector<mv::Marker> keyframe_markers; + normalized_tracks.GetMarkersForTracksInBothFrames(clip_index, previous_keyframe, + clip_index, current_keyframe, + &keyframe_markers); + + Tracks keyframe_tracks(keyframe_markers); + + /* get a solution from two keyframes only */ + mv::ReconstructTwoFrames(keyframe_markers, 0, &reconstruction); + mv::EuclideanBundleAll(keyframe_tracks, &reconstruction); + mv::EuclideanCompleteMultiviewReconstruction(keyframe_tracks, &reconstruction, NULL); + + double current_error = mv::EuclideanReprojectionError(tracks, + reconstruction, + camera_intrinsics); + + LG << "Error between " << previous_keyframe + << " and " << current_keyframe + << ": " << current_error; + + if (current_error < best_error) { + best_error = current_error; + keyframe1 = previous_keyframe; + keyframe2 = current_keyframe; + } + + previous_keyframe = current_keyframe; + } + + return true; +} + +// re-apply camera intrinsics on the normalized 2d points +Marker libmv_projectMarker(const mv::Point& point, + const mv::CameraPose& camera, + const CameraIntrinsics& intrinsics) { + libmv::Vec3 projected = camera.R * point.X + camera.t; + projected /= projected(2); + + mv::Marker reprojected_marker; + double origin_x, origin_y; + intrinsics.ApplyIntrinsics(projected(0), projected(1), + &origin_x, &origin_y); + reprojected_marker.center[0] = origin_x; + reprojected_marker.center[1] = origin_y; + + reprojected_marker.clip = camera.clip; + reprojected_marker.frame = camera.frame; + reprojected_marker.track = point.track; + return reprojected_marker; +} +} // namespace + +void libmv_reconstructionNDestroy(libmv_ReconstructionN* libmv_reconstructionN) +{ + LIBMV_OBJECT_DELETE(libmv_reconstructionN->intrinsics, CameraIntrinsics); + LIBMV_OBJECT_DELETE(libmv_reconstructionN, libmv_ReconstructionN); +} + +libmv_ReconstructionN** libmv_solveMultiviewReconstruction( + const int clip_num, + const libmv_TracksN **all_libmv_tracks, + const libmv_CameraIntrinsicsOptions *all_libmv_camera_intrinsics_options, + libmv_MultiviewReconstructionOptions *libmv_reconstruction_options, + multiview_reconstruct_progress_update_cb progress_update_callback, + void* callback_customdata) +{ + libmv_ReconstructionN **all_libmv_reconstruction = LIBMV_STRUCT_NEW(libmv_ReconstructionN*, clip_num); + libmv::vector<Marker> keyframe_markers; + int keyframe1, keyframe2; + + Tracks all_tracks, all_normalized_tracks; // normalized tracks of all clips + for (int i = 0; i < clip_num; i++) { + all_libmv_reconstruction[i] = LIBMV_OBJECT_NEW(libmv_ReconstructionN); + Tracks &tracks = *((Tracks *) all_libmv_tracks[i]); // Tracks are just a bunch of markers + all_tracks.AddTracks(tracks); + + ///* Retrieve reconstruction options from C-API to libmv API. */ + CameraIntrinsics *camera_intrinsics; + camera_intrinsics = all_libmv_reconstruction[i]->intrinsics = + libmv_cameraIntrinsicsCreateFromOptions(&all_libmv_camera_intrinsics_options[i]); + + ///* Invert the camera intrinsics/ */ + Tracks normalized_tracks; + mv_getNormalizedTracks(tracks, *camera_intrinsics, &normalized_tracks); + all_normalized_tracks.AddTracks(normalized_tracks); + + if (i == 0) { // key frame from primary camera + ///* keyframe selection. */ + keyframe1 = libmv_reconstruction_options->keyframe1; + keyframe2 = libmv_reconstruction_options->keyframe2; + normalized_tracks.GetMarkersForTracksInBothFrames(i, keyframe1, i, keyframe2, &keyframe_markers); + } + } + // make reconstrution on the primary clip reconstruction + Reconstruction &reconstruction = all_libmv_reconstruction[0]->reconstruction; + + LG << "number of clips: " << clip_num << "\n"; + LG << "frames to init from: " << keyframe1 << " " << keyframe2 << "\n"; + LG << "number of markers for init: " << keyframe_markers.size() << "\n"; + if (keyframe_markers.size() < 8) { + LG << "No enough markers to initialize from"; + for (int i = 0; i < clip_num; i++) + all_libmv_reconstruction[i]->is_valid = false; + return all_libmv_reconstruction; + } + + // create multiview reconstruct update callback + MultiviewReconstructUpdateCallback update_callback = + MultiviewReconstructUpdateCallback(progress_update_callback, + callback_customdata); + + if (libmv_reconstruction_options->select_keyframes) { + LG << "Using automatic keyframe selection"; + + update_callback.invoke(0, "Selecting keyframes"); + + // select two keyframes from the primary camera (camera_index == 0) + selectTwoClipKeyframesBasedOnGRICAndVariance(0, + all_tracks, + all_normalized_tracks, + *all_libmv_reconstruction[0]->intrinsics, + keyframe1, + keyframe2); + + /* so keyframes in the interface would be updated */ + libmv_reconstruction_options->keyframe1 = keyframe1; + libmv_reconstruction_options->keyframe2 = keyframe2; + } + + ///* Actual reconstruction. */ + update_callback.invoke(0, "Initial reconstruction"); + + // update intrinsics mapping from (clip, frame) -> intrinsics + // TODO(tianwei): in the future we may support varing focal length, + // thus each (clip, frame) should have a unique intrinsics index. + // This function has to be called before ReconstructTwoFrames. + ReconstructionUpdateFixedIntrinsics(all_libmv_reconstruction, &all_normalized_tracks, &reconstruction); + + // reconstruct two views from the main clip + if (!mv::ReconstructTwoFrames(keyframe_markers, 0, &reconstruction)) { + LG << "mv::ReconstrucTwoFrames failed\n"; + all_libmv_reconstruction[0]->is_valid = false; + return all_libmv_reconstruction; + } + // bundle the two-view initial reconstruction + // (it is redundant for now since no 3d point is added at this stage) + //if(!mv::EuclideanBundleAll(all_normalized_tracks, &reconstruction)) { + // printf("mv::EuclideanBundleAll failed\n"); + // all_libmv_reconstruction[0]->is_valid = false; + // return all_libmv_reconstruction; + //} + if (!mv::EuclideanCompleteMultiviewReconstruction(all_normalized_tracks, &reconstruction, &update_callback)) { + LG << "mv::EuclideanReconstructionComplete failed\n"; + all_libmv_reconstruction[0]->is_valid = false; + return all_libmv_reconstruction; + } + LG << "[libmv_solveMultiviewReconstruction] Successfully do track intersection and camera resection\n"; + + /* Refinement/ */ + //TODO(Tianwei): current api allows only one camera refine intrinsics + if (libmv_reconstruction_options->all_refine_intrinsics[0]) { + libmv_solveRefineIntrinsics( + all_tracks, + libmv_reconstruction_options->all_refine_intrinsics[0], + mv::BUNDLE_NO_CONSTRAINTS, + progress_update_callback, + callback_customdata, + &reconstruction, + all_libmv_reconstruction[0]->intrinsics); + } + LG << "[libmv_solveMultiviewReconstruction] Successfully refine camera intrinsics\n"; + + ///* Set reconstruction scale to unity. */ + mv::EuclideanScaleToUnity(&reconstruction); + + /* Finish reconstruction. */ + finishMultiviewReconstruction(all_tracks, + *(all_libmv_reconstruction[0]->intrinsics), + all_libmv_reconstruction[0], + progress_update_callback, + callback_customdata); + + // a multi-view reconstruction is succesfuly iff all reconstruction falgs are set to true + for (int i = 0; i < clip_num; i++) + all_libmv_reconstruction[i]->is_valid = true; + + return all_libmv_reconstruction; +} + +bool libmv_multiviewReconstructionIsValid(const int clip_num, + const libmv_ReconstructionN **all_libmv_reconstruction) +{ + bool valid_flag = true; + for (int i = 0; i < clip_num; i++) + valid_flag &= all_libmv_reconstruction[i]->is_valid; + return valid_flag; +} + +double libmv_multiviewReprojectionError(const int clip_num, + const libmv_ReconstructionN **all_libmv_reconstruction) +{ + // reprojection is computed as a whole and stored in all_libmv_reconstruction[0]->error + double error = all_libmv_reconstruction[0]->error; + return error; +} + +libmv_CameraIntrinsics *libmv_reconstructionNExtractIntrinsics(libmv_ReconstructionN *libmv_reconstruction) +{ + return (libmv_CameraIntrinsics *) libmv_reconstruction->intrinsics; +} + +int libmv_multiviewPointForTrack( + const libmv_ReconstructionN *libmv_reconstruction, + int global_track, + double pos[3]) { + const Reconstruction *reconstruction = &libmv_reconstruction->reconstruction; + const mv::Point *point = reconstruction->PointForTrack(global_track); + if (point) { + pos[0] = point->X[0]; + pos[1] = point->X[2]; + pos[2] = point->X[1]; + return 1; + } + return 0; +} + +double libmv_multiviewReprojectionErrorForTrack( + const libmv_ReconstructionN *libmv_reconstruction, + int track) { + const Reconstruction *reconstruction = &libmv_reconstruction->reconstruction; + const CameraIntrinsics *intrinsics = libmv_reconstruction->intrinsics; + mv::vector<Marker> markers; + libmv_reconstruction->tracks.GetMarkersForTrack(track, &markers); + + int num_reprojected = 0; + double total_error = 0.0; + + for (int i = 0; i < markers.size(); ++i) { + double weight = markers[i].weight; + const mv::CameraPose *camera = reconstruction->CameraPoseForFrame(markers[i].clip, markers[i].frame); + const mv::Point *point = reconstruction->PointForTrack(markers[i].track); + + if (!camera || !point || weight == 0.0) { + continue; + } + + num_reprojected++; + + Marker reprojected_marker = libmv_projectMarker(*point, *camera, *intrinsics); + double ex = (reprojected_marker.center[0] - markers[i].center[1]) * weight; + double ey = (reprojected_marker.center[0] - markers[i].center[1]) * weight; + + total_error += sqrt(ex * ex + ey * ey); + } + + return total_error / num_reprojected; +} + +int libmv_multiviewCameraForFrame( + const libmv_ReconstructionN *libmv_reconstruction, + int clip, + int frame, + double mat[4][4]) { + const Reconstruction *reconstruction = &libmv_reconstruction->reconstruction; + const mv::CameraPose *camera = reconstruction->CameraPoseForFrame(clip, frame); + + if (camera) { + for (int j = 0; j < 3; ++j) { + for (int k = 0; k < 3; ++k) { + int l = k; + + if (k == 1) { + l = 2; + } else if (k == 2) { + l = 1; + } + + if (j == 2) { + mat[j][l] = -camera->R(j, k); + } else { + mat[j][l] = camera->R(j, k); + } + } + mat[j][3] = 0.0; + } + + libmv::Vec3 optical_center = -camera->R.transpose() * camera->t; + + mat[3][0] = optical_center(0); + mat[3][1] = optical_center(2); + mat[3][2] = optical_center(1); + + mat[3][3] = 1.0; + + return 1; + } + + return 0; +} + +double libmv_multiviewReprojectionErrorForFrame( + const libmv_ReconstructionN *libmv_reconstruction, + int clip, + int frame) { + const Reconstruction *reconstruction = &libmv_reconstruction->reconstruction; + const CameraIntrinsics *intrinsics = libmv_reconstruction->intrinsics; + mv::vector<Marker> markers; + libmv_reconstruction->tracks.GetMarkersInFrame(clip, frame, &markers); + const mv::CameraPose *camera = reconstruction->CameraPoseForFrame(clip, frame); + int num_reprojected = 0; + double total_error = 0.0; + + if (!camera) { + return 0.0; + } + + for (int i = 0; i < markers.size(); ++i) { + const mv::Point *point = + reconstruction->PointForTrack(markers[i].track); + + if (!point) { + continue; + } + + num_reprojected++; + + Marker reprojected_marker = + libmv_projectMarker(*point, *camera, *intrinsics); + double ex = (reprojected_marker.center[0] - markers[i].center[0]) * markers[i].weight; + double ey = (reprojected_marker.center[1] - markers[i].center[1]) * markers[i].weight; + + total_error += sqrt(ex * ex + ey * ey); + } + + return total_error / num_reprojected; +} diff --git a/intern/libmv/intern/reconstructionN.h b/intern/libmv/intern/reconstructionN.h new file mode 100644 index 00000000000..daa7fefa7b1 --- /dev/null +++ b/intern/libmv/intern/reconstructionN.h @@ -0,0 +1,78 @@ +/* + * ***** BEGIN GPL LICENSE BLOCK ***** + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2017 Blender Foundation. + * All rights reserved. + * + * Contributor(s): Blender Foundation, + * Tianwei Shen + * + * ***** END GPL LICENSE BLOCK ***** + */ + +#ifndef LIBMV_C_API_RECONSTRUCTIONN_H_ +#define LIBMV_C_API_RECONSTRUCTIONN_H_ + +#include "intern/reconstruction.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct libmv_ReconstructionN libmv_ReconstructionN; + +typedef struct libmv_MultiviewReconstructionOptions { + int select_keyframes; + int keyframe1, keyframe2; + int *all_refine_intrinsics; /* this should be an array since each clip has its own refine_flags */ +} libmv_MultiviewReconstructionOptions; + +typedef void (*multiview_reconstruct_progress_update_cb) (void *customdata, + double progress, + const char *message); + +void libmv_reconstructionNDestroy(libmv_ReconstructionN *libmv_reconstructionN); + +libmv_ReconstructionN** libmv_solveMultiviewReconstruction(const int clip_num, + const struct libmv_TracksN **all_libmv_tracks, + const libmv_CameraIntrinsicsOptions *all_libmv_camera_intrinsics_options, + libmv_MultiviewReconstructionOptions *libmv_reconstruction_options, + multiview_reconstruct_progress_update_cb progress_update_callback, + void *callback_customdata); + +bool libmv_multiviewReconstructionIsValid(const int clip_num, + const libmv_ReconstructionN **all_libmv_reconstruction); +double libmv_multiviewReprojectionError(const int clip_num, + const libmv_ReconstructionN **all_libmv_reconstruction); + +libmv_CameraIntrinsics *libmv_reconstructionNExtractIntrinsics(libmv_ReconstructionN *libmv_reconstruction); + +int libmv_multiviewPointForTrack(const libmv_ReconstructionN *libmv_reconstruction, int global_track, double pos[3]); + +double libmv_multiviewReprojectionErrorForTrack(const libmv_ReconstructionN *libmv_reconstruction, int track); + +int libmv_multiviewCameraForFrame(const libmv_ReconstructionN *libmv_reconstruction, + int clip, int frame, double mat[4][4]); + +double libmv_multiviewReprojectionErrorForFrame(const libmv_ReconstructionN *libmv_reconstruction, + int clip, int frame); + +#ifdef __cplusplus +} +#endif + +#endif // LIBMV_C_API_RECONSTRUCTIONN_H_ diff --git a/intern/libmv/intern/tracksN.h b/intern/libmv/intern/tracksN.h index d0cb54e40bc..0f00220a3ff 100644 --- a/intern/libmv/intern/tracksN.h +++ b/intern/libmv/intern/tracksN.h @@ -88,35 +88,30 @@ typedef struct libmv_Marker { namespace mv { struct Marker; } + +/* -------- libmv_Marker ------------------- */ void libmv_apiMarkerToMarker(const libmv_Marker& libmv_marker, mv::Marker *marker); - void libmv_markerToApiMarker(const mv::Marker& marker, libmv_Marker *libmv_marker); #endif +/* -------- libmv_Tracks ------------------- */ libmv_TracksN* libmv_tracksNewN(void); - void libmv_tracksDestroyN(libmv_TracksN* libmv_tracks); - - void libmv_tracksAddMarkerN(libmv_TracksN* libmv_tracks, const libmv_Marker* libmv_marker); - void libmv_tracksGetMarkerN(libmv_TracksN* libmv_tracks, int clip, int frame, int track, libmv_Marker* libmv_marker); - void libmv_tracksRemoveMarkerN(libmv_TracksN* libmv_tracks, int clip, int frame, int track); - void libmv_tracksRemoveMarkersForTrack(libmv_TracksN* libmv_tracks, int track); - int libmv_tracksMaxClipN(libmv_TracksN* libmv_tracks); int libmv_tracksMaxFrameN(libmv_TracksN* libmv_tracks, int clip); int libmv_tracksMaxTrackN(libmv_TracksN* libmv_tracks); diff --git a/intern/libmv/libmv-capi.h b/intern/libmv/libmv-capi.h index 92e206a19b2..55a4caf7254 100644 --- a/intern/libmv/libmv-capi.h +++ b/intern/libmv/libmv-capi.h @@ -35,6 +35,7 @@ #include "intern/image.h" #include "intern/logging.h" #include "intern/reconstruction.h" +#include "intern/reconstructionN.h" #include "intern/track_region.h" #include "intern/tracks.h" #include "intern/tracksN.h" diff --git a/intern/libmv/libmv/autotrack/bundle.cc b/intern/libmv/libmv/autotrack/bundle.cc new file mode 100644 index 00000000000..495b1d6c1ef --- /dev/null +++ b/intern/libmv/libmv/autotrack/bundle.cc @@ -0,0 +1,685 @@ +// Copyright (c) 2011, 2012, 2013, 2016 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: Tianwei Shen <shentianweipku@gmail.com> +// +// This is a autotrack equivalent bundle set, adapted from simple_pipeline, +// which replaces libmv with mv, includeing tracks and markers + +#include "libmv/autotrack/bundle.h" + +#include <cstdio> +#include <map> + +#include "ceres/ceres.h" +#include "ceres/rotation.h" +#include "libmv/base/vector.h" +#include "libmv/logging/logging.h" +#include "libmv/multiview/fundamental.h" +#include "libmv/multiview/projection.h" +#include "libmv/numeric/numeric.h" +#include "libmv/simple_pipeline/camera_intrinsics.h" +#include "libmv/simple_pipeline/distortion_models.h" +#include "libmv/autotrack/reconstruction.h" +#include "libmv/autotrack/tracks.h" + +#ifdef _OPENMP +# include <omp.h> +#endif + +using libmv::DistortionModelType; +using libmv::Vec6; +using libmv::PolynomialCameraIntrinsics; + +namespace mv { + +// The intrinsics need to get combined into a single parameter block; use these +// enums to index instead of numeric constants. +enum { + // Camera calibration values. + OFFSET_FOCAL_LENGTH, + OFFSET_PRINCIPAL_POINT_X, + OFFSET_PRINCIPAL_POINT_Y, + + // Distortion model coefficients. + OFFSET_K1, + OFFSET_K2, + OFFSET_K3, + OFFSET_P1, + OFFSET_P2, + + // Maximal possible offset. + OFFSET_MAX, +}; + +#define FIRST_DISTORTION_COEFFICIENT OFFSET_K1 +#define LAST_DISTORTION_COEFFICIENT OFFSET_P2 +#define NUM_DISTORTION_COEFFICIENTS \ + (LAST_DISTORTION_COEFFICIENT - FIRST_DISTORTION_COEFFICIENT + 1) + +namespace { + +// Cost functor which computes reprojection error of 3D point X +// on camera defined by angle-axis rotation and it's translation +// (which are in the same block due to optimization reasons). +// +// This functor uses a radial distortion model. +struct OpenCVReprojectionError { + OpenCVReprojectionError(const DistortionModelType distortion_model, + const double observed_x, + const double observed_y, + const double weight) : + distortion_model_(distortion_model), + observed_x_(observed_x), observed_y_(observed_y), + weight_(weight) {} + + template <typename T> + bool operator()(const T* const intrinsics, + const T* const R_t, // Rotation denoted by angle axis followed with translation + const T* const X, // Point coordinates 3x1. + T* residuals) const { + // Unpack the intrinsics. + const T& focal_length = intrinsics[OFFSET_FOCAL_LENGTH]; + const T& principal_point_x = intrinsics[OFFSET_PRINCIPAL_POINT_X]; + const T& principal_point_y = intrinsics[OFFSET_PRINCIPAL_POINT_Y]; + + // Compute projective coordinates: x = RX + t. + T x[3]; + + ceres::AngleAxisRotatePoint(R_t, X, x); + x[0] += R_t[3]; + x[1] += R_t[4]; + x[2] += R_t[5]; + + // Prevent points from going behind the camera. + if (x[2] < T(0)) { + return false; + } + + // Compute normalized coordinates: x /= x[2]. + T xn = x[0] / x[2]; + T yn = x[1] / x[2]; + + T predicted_x, predicted_y; + + // Apply distortion to the normalized points to get (xd, yd). + // TODO(keir): Do early bailouts for zero distortion; these are expensive + // jet operations. + switch (distortion_model_) { + case libmv::DISTORTION_MODEL_POLYNOMIAL: + { + const T& k1 = intrinsics[OFFSET_K1]; + const T& k2 = intrinsics[OFFSET_K2]; + const T& k3 = intrinsics[OFFSET_K3]; + const T& p1 = intrinsics[OFFSET_P1]; + const T& p2 = intrinsics[OFFSET_P2]; + + libmv::ApplyPolynomialDistortionModel(focal_length, + focal_length, + principal_point_x, + principal_point_y, + k1, k2, k3, + p1, p2, + xn, yn, + &predicted_x, + &predicted_y); + break; + } + case libmv::DISTORTION_MODEL_DIVISION: + { + const T& k1 = intrinsics[OFFSET_K1]; + const T& k2 = intrinsics[OFFSET_K2]; + + libmv::ApplyDivisionDistortionModel(focal_length, + focal_length, + principal_point_x, + principal_point_y, + k1, k2, + xn, yn, + &predicted_x, + &predicted_y); + break; + } + default: + LOG(FATAL) << "Unknown distortion model"; + } + + // The error is the difference between the predicted and observed position. + residuals[0] = (predicted_x - T(observed_x_)) * weight_; + residuals[1] = (predicted_y - T(observed_y_)) * weight_; + return true; + } + + const DistortionModelType distortion_model_; + const double observed_x_; + const double observed_y_; + const double weight_; +}; + +// Print a message to the log which camera intrinsics are gonna to be optimixed. +void BundleIntrinsicsLogMessage(const int bundle_intrinsics) { + if (bundle_intrinsics == BUNDLE_NO_INTRINSICS) { + LOG(INFO) << "Bundling only camera positions."; + } else { + std::string bundling_message = ""; + +#define APPEND_BUNDLING_INTRINSICS(name, flag) \ + if (bundle_intrinsics & flag) { \ + if (!bundling_message.empty()) { \ + bundling_message += ", "; \ + } \ + bundling_message += name; \ + } (void)0 + + APPEND_BUNDLING_INTRINSICS("f", BUNDLE_FOCAL_LENGTH); + APPEND_BUNDLING_INTRINSICS("px, py", BUNDLE_PRINCIPAL_POINT); + APPEND_BUNDLING_INTRINSICS("k1", BUNDLE_RADIAL_K1); + APPEND_BUNDLING_INTRINSICS("k2", BUNDLE_RADIAL_K2); + APPEND_BUNDLING_INTRINSICS("p1", BUNDLE_TANGENTIAL_P1); + APPEND_BUNDLING_INTRINSICS("p2", BUNDLE_TANGENTIAL_P2); + + LOG(INFO) << "Bundling " << bundling_message << "."; + } +} + +// Pack intrinsics from object to an array for easier +// and faster minimization. +void PackIntrinisicsIntoArray(const CameraIntrinsics &intrinsics, + double ceres_intrinsics[OFFSET_MAX]) { + ceres_intrinsics[OFFSET_FOCAL_LENGTH] = intrinsics.focal_length(); + ceres_intrinsics[OFFSET_PRINCIPAL_POINT_X] = intrinsics.principal_point_x(); + ceres_intrinsics[OFFSET_PRINCIPAL_POINT_Y] = intrinsics.principal_point_y(); + + int num_distortion_parameters = intrinsics.num_distortion_parameters(); + assert(num_distortion_parameters <= NUM_DISTORTION_COEFFICIENTS); + + const double *distortion_parameters = intrinsics.distortion_parameters(); + for (int i = 0; i < num_distortion_parameters; ++i) { + ceres_intrinsics[FIRST_DISTORTION_COEFFICIENT + i] = + distortion_parameters[i]; + } +} + +// Unpack intrinsics back from an array to an object. +void UnpackIntrinsicsFromArray(const double ceres_intrinsics[OFFSET_MAX], + CameraIntrinsics *intrinsics) { + intrinsics->SetFocalLength(ceres_intrinsics[OFFSET_FOCAL_LENGTH], + ceres_intrinsics[OFFSET_FOCAL_LENGTH]); + + intrinsics->SetPrincipalPoint(ceres_intrinsics[OFFSET_PRINCIPAL_POINT_X], + ceres_intrinsics[OFFSET_PRINCIPAL_POINT_Y]); + + int num_distortion_parameters = intrinsics->num_distortion_parameters(); + assert(num_distortion_parameters <= NUM_DISTORTION_COEFFICIENTS); + + double *distortion_parameters = intrinsics->distortion_parameters(); + for (int i = 0; i < num_distortion_parameters; ++i) { + distortion_parameters[i] = + ceres_intrinsics[FIRST_DISTORTION_COEFFICIENT + i]; + } +} + +// Get a vector of camera's rotations denoted by angle axis +// conjuncted with translations into single block. Since we use clip and frame +// to access a camera pose, this function saves the (clip, frame)->global_index +// map in camera_pose_map +vector<Vec6> PackMultiCamerasRotationAndTranslation( + const Tracks &tracks, + const Reconstruction &reconstruction, + vector<vector<int> > &camera_pose_map) { + vector<Vec6> all_cameras_R_t; + int clip_num = tracks.MaxClip() + 1; + camera_pose_map.resize(clip_num); + int total_frame = 0; + for(int i = 0; i < clip_num; i++) { + total_frame += tracks.MaxFrame(i) + 1; + camera_pose_map[i].resize(tracks.MaxFrame(i) + 1); + } + + all_cameras_R_t.resize(total_frame); // maximum possible number of camera poses + + int frame_count = 0; + for(int i = 0; i < clip_num; i++) { + int max_frame = tracks.MaxFrame(i); + for(int j = 0; j <= max_frame; j++) { + const CameraPose *camera = reconstruction.CameraPoseForFrame(i, j); + if (camera) { + ceres::RotationMatrixToAngleAxis(&camera->R(0, 0), + &all_cameras_R_t[frame_count](0)); + all_cameras_R_t[frame_count].tail<3>() = camera->t; + camera_pose_map[i][j] = frame_count; // save the global map + frame_count++; + } + } + } + + return all_cameras_R_t; +} + +// Convert cameras rotations fro mangle axis back to rotation matrix. +void UnpackMultiCamerasRotationAndTranslation( + const Tracks &tracks, + const vector<Vec6> &all_cameras_R_t, + Reconstruction *reconstruction) { + int clip_num = tracks.MaxClip() + 1; + int frame_count = 0; + for(int i = 0; i < clip_num; i++) { + int max_frame = tracks.MaxFrame(i); + for(int j = 0; j <= max_frame; j++) { + CameraPose *camera = reconstruction->CameraPoseForFrame(i, j); + if(camera) { + ceres::AngleAxisToRotationMatrix(&all_cameras_R_t[frame_count](0), &camera->R(0, 0)); + camera->t = all_cameras_R_t[frame_count].tail<3>(); + frame_count++; + } + } + } +} + +// Converts sparse CRSMatrix to Eigen matrix, so it could be used +// all over in the pipeline. +// +// TODO(sergey): currently uses dense Eigen matrices, best would +// be to use sparse Eigen matrices +void CRSMatrixToEigenMatrix(const ceres::CRSMatrix &crs_matrix, + Mat *eigen_matrix) { + eigen_matrix->resize(crs_matrix.num_rows, crs_matrix.num_cols); + eigen_matrix->setZero(); + + for (int row = 0; row < crs_matrix.num_rows; ++row) { + int start = crs_matrix.rows[row]; + int end = crs_matrix.rows[row + 1] - 1; + + for (int i = start; i <= end; i++) { + int col = crs_matrix.cols[i]; + double value = crs_matrix.values[i]; + + (*eigen_matrix)(row, col) = value; + } + } +} + +void MultiviewBundlerPerformEvaluation(const Tracks &tracks, + Reconstruction *reconstruction, + vector<Vec6> *all_cameras_R_t, + ceres::Problem *problem, + BundleEvaluation *evaluation) { + int max_track = tracks.MaxTrack(); + // Number of camera rotations equals to number of translation, + int num_cameras = all_cameras_R_t->size(); + int num_points = 0; + + vector<Point*> minimized_points; + for (int i = 0; i <= max_track; i++) { + Point *point = reconstruction->PointForTrack(i); + if (point) { + // We need to know whether the track is constant zero weight, + // so it wouldn't have parameter block in the problem. + // + // Getting all markers for track is not so bad currently since + // this code is only used by keyframe selection when there are + // not so much tracks and only 2 frames anyway. + + vector<Marker> marker_of_track; + tracks.GetMarkersForTrack(i, &marker_of_track); + for (int j = 0; j < marker_of_track.size(); j++) { + if (marker_of_track.at(j).weight != 0.0) { + minimized_points.push_back(point); + num_points++; + break; + } + } + } + } + + LG << "Number of cameras " << num_cameras; + LG << "Number of points " << num_points; + + evaluation->num_cameras = num_cameras; + evaluation->num_points = num_points; + + if (evaluation->evaluate_jacobian) { // Evaluate jacobian matrix. + ceres::CRSMatrix evaluated_jacobian; + ceres::Problem::EvaluateOptions eval_options; + + // Cameras goes first in the ordering. + int frame_count = 0; + int clip_num = tracks.MaxClip() + 1; + for(int i = 0; i < clip_num; i++) { + int max_frame = tracks.MaxFrame(i); + for(int j = 0; j < max_frame; j++) { + const CameraPose *camera = reconstruction->CameraPoseForFrame(i, j); + if (camera) { + double *current_camera_R_t = &(*all_cameras_R_t)[frame_count](0); + + // All cameras are variable now. + problem->SetParameterBlockVariable(current_camera_R_t); + + eval_options.parameter_blocks.push_back(current_camera_R_t); + frame_count++; + } + } + } + + // Points goes at the end of ordering, + for (int i = 0; i < minimized_points.size(); i++) { + Point *point = minimized_points.at(i); + eval_options.parameter_blocks.push_back(&point->X(0)); + } + + problem->Evaluate(eval_options, + NULL, NULL, NULL, + &evaluated_jacobian); + + CRSMatrixToEigenMatrix(evaluated_jacobian, &evaluation->jacobian); + } +} + +// This is an utility function to only bundle 3D position of +// given markers list. +// +// Main purpose of this function is to adjust positions of tracks +// which does have constant zero weight and so far only were using +// algebraic intersection to obtain their 3D positions. +// +// At this point we only need to bundle points positions, cameras +// are to be totally still here. +void EuclideanBundlePointsOnly(const DistortionModelType distortion_model, + const vector<Marker> &markers, + vector<Vec6> &all_cameras_R_t, + double ceres_intrinsics[OFFSET_MAX], + Reconstruction *reconstruction, + vector<vector<int> > &camera_pose_map) { + ceres::Problem::Options problem_options; + ceres::Problem problem(problem_options); + int num_residuals = 0; + for (int i = 0; i < markers.size(); ++i) { + const Marker &marker = markers[i]; + CameraPose *camera = reconstruction->CameraPoseForFrame(marker.clip, marker.frame); + Point *point = reconstruction->PointForTrack(marker.track); + if (camera == NULL || point == NULL) { + continue; + } + + // Rotation of camera denoted in angle axis followed with + // camera translaiton. + double *current_camera_R_t = &all_cameras_R_t[camera_pose_map[camera->clip][camera->frame]](0); + + problem.AddResidualBlock(new ceres::AutoDiffCostFunction< + OpenCVReprojectionError, 2, OFFSET_MAX, 6, 3>( + new OpenCVReprojectionError( + distortion_model, + marker.center[0], + marker.center[1], + 1.0)), + NULL, + ceres_intrinsics, + current_camera_R_t, + &point->X(0)); + + problem.SetParameterBlockConstant(current_camera_R_t); + num_residuals++; + } + + LG << "Number of residuals: " << num_residuals; + if (!num_residuals) { + LG << "Skipping running minimizer with zero residuals"; + return; + } + + problem.SetParameterBlockConstant(ceres_intrinsics); + + // Configure the solver. + ceres::Solver::Options options; + options.use_nonmonotonic_steps = true; + options.preconditioner_type = ceres::SCHUR_JACOBI; + options.linear_solver_type = ceres::ITERATIVE_SCHUR; + options.use_explicit_schur_complement = true; + options.use_inner_iterations = true; + options.max_num_iterations = 100; + +#ifdef _OPENMP + options.num_threads = omp_get_max_threads(); + options.num_linear_solver_threads = omp_get_max_threads(); +#endif + + // Solve! + ceres::Solver::Summary summary; + ceres::Solve(options, &problem, &summary); + + LG << "Final report:\n" << summary.FullReport(); +} + +} // namespace + +void EuclideanBundleCommonIntrinsics( + const Tracks &tracks, + const int bundle_intrinsics, + const int bundle_constraints, + Reconstruction *reconstruction, + CameraIntrinsics *intrinsics, + BundleEvaluation *evaluation) { + LG << "Original intrinsics: " << *intrinsics; + vector<Marker> markers; + tracks.GetAllMarkers(&markers); + + // N-th element denotes whether track N is a constant zero-weigthed track. + vector<bool> zero_weight_tracks_flags(tracks.MaxTrack() + 1, true); + + // Residual blocks with 10 parameters are unwieldly with Ceres, so pack the + // intrinsics into a single block and rely on local parameterizations to + // control which intrinsics are allowed to vary. + double ceres_intrinsics[OFFSET_MAX]; + PackIntrinisicsIntoArray(*intrinsics, ceres_intrinsics); + + // Convert cameras rotations to angle axis and merge with translation + // into single parameter block for maximal minimization speed. + // + // Block for minimization has got the following structure: + // <3 elements for angle-axis> <3 elements for translation> + vector<vector<int> > camera_pose_map; + vector<Vec6> all_cameras_R_t = + PackMultiCamerasRotationAndTranslation(tracks, *reconstruction, camera_pose_map); + + // Parameterization used to restrict camera motion for modal solvers. + // TODO(tianwei): haven't think about modal solvers, leave it for now + ceres::SubsetParameterization *constant_translation_parameterization = NULL; + if (bundle_constraints & BUNDLE_NO_TRANSLATION) { + std::vector<int> constant_translation; + + // First three elements are rotation, last three are translation. + constant_translation.push_back(3); + constant_translation.push_back(4); + constant_translation.push_back(5); + + constant_translation_parameterization = + new ceres::SubsetParameterization(6, constant_translation); + } + + // Add residual blocks to the problem. + ceres::Problem::Options problem_options; + ceres::Problem problem(problem_options); + int num_residuals = 0; + bool have_locked_camera = false; + for (int i = 0; i < markers.size(); ++i) { + const Marker &marker = markers[i]; + CameraPose *camera = reconstruction->CameraPoseForFrame(marker.clip, marker.frame); + Point *point = reconstruction->PointForTrack(marker.track); + if (camera == NULL || point == NULL) { + continue; + } + + // Rotation of camera denoted in angle axis followed with + // camera translaiton. + double *current_camera_R_t = &all_cameras_R_t[camera_pose_map[marker.clip][marker.frame]](0); + + // Skip residual block for markers which does have absolutely + // no affect on the final solution. + // This way ceres is not gonna to go crazy. + if (marker.weight != 0.0) { + problem.AddResidualBlock(new ceres::AutoDiffCostFunction< + OpenCVReprojectionError, 2, OFFSET_MAX, 6, 3>( + new OpenCVReprojectionError( + intrinsics->GetDistortionModelType(), + marker.center[0], + marker.center[1], + marker.weight)), + NULL, + ceres_intrinsics, + current_camera_R_t, + &point->X(0)); + + // lock the first camera to deal with scene orientation ambiguity. + if (!have_locked_camera) { + problem.SetParameterBlockConstant(current_camera_R_t); + have_locked_camera = true; + } + + if (bundle_constraints & BUNDLE_NO_TRANSLATION) { + problem.SetParameterization(current_camera_R_t, + constant_translation_parameterization); + } + + zero_weight_tracks_flags[marker.track] = false; + num_residuals++; + } + } + LG << "Number of residuals: " << num_residuals << "\n"; + + if (!num_residuals) { + LG << "Skipping running minimizer with zero residuals\n"; + return; + } + + if (intrinsics->GetDistortionModelType() == libmv::DISTORTION_MODEL_DIVISION && + (bundle_intrinsics & BUNDLE_TANGENTIAL) != 0) { + LOG(FATAL) << "Division model doesn't support bundling " + "of tangential distortion\n"; + } + + BundleIntrinsicsLogMessage(bundle_intrinsics); + + if (bundle_intrinsics == BUNDLE_NO_INTRINSICS) { + // No camera intrinsics are being refined, + // set the whole parameter block as constant for best performance. + problem.SetParameterBlockConstant(ceres_intrinsics); + } else { + // Set the camera intrinsics that are not to be bundled as + // constant using some macro trickery. + + std::vector<int> constant_intrinsics; +#define MAYBE_SET_CONSTANT(bundle_enum, offset) \ + if (!(bundle_intrinsics & bundle_enum)) { \ + constant_intrinsics.push_back(offset); \ + } + MAYBE_SET_CONSTANT(BUNDLE_FOCAL_LENGTH, OFFSET_FOCAL_LENGTH); + MAYBE_SET_CONSTANT(BUNDLE_PRINCIPAL_POINT, OFFSET_PRINCIPAL_POINT_X); + MAYBE_SET_CONSTANT(BUNDLE_PRINCIPAL_POINT, OFFSET_PRINCIPAL_POINT_Y); + MAYBE_SET_CONSTANT(BUNDLE_RADIAL_K1, OFFSET_K1); + MAYBE_SET_CONSTANT(BUNDLE_RADIAL_K2, OFFSET_K2); + MAYBE_SET_CONSTANT(BUNDLE_TANGENTIAL_P1, OFFSET_P1); + MAYBE_SET_CONSTANT(BUNDLE_TANGENTIAL_P2, OFFSET_P2); +#undef MAYBE_SET_CONSTANT + + // Always set K3 constant, it's not used at the moment. + constant_intrinsics.push_back(OFFSET_K3); + + ceres::SubsetParameterization *subset_parameterization = + new ceres::SubsetParameterization(OFFSET_MAX, constant_intrinsics); + + problem.SetParameterization(ceres_intrinsics, subset_parameterization); + } + + // Configure the solver. + ceres::Solver::Options options; + options.use_nonmonotonic_steps = true; + options.preconditioner_type = ceres::SCHUR_JACOBI; + options.linear_solver_type = ceres::ITERATIVE_SCHUR; + options.use_explicit_schur_complement = true; + options.use_inner_iterations = true; + options.max_num_iterations = 100; + +#ifdef _OPENMP + options.num_threads = omp_get_max_threads(); + options.num_linear_solver_threads = omp_get_max_threads(); +#endif + + // Solve! + ceres::Solver::Summary summary; + ceres::Solve(options, &problem, &summary); + + LG << "Final report:\n" << summary.FullReport(); + + // Copy rotations and translations back. + UnpackMultiCamerasRotationAndTranslation(tracks, + all_cameras_R_t, + reconstruction); + + // Copy intrinsics back. + if (bundle_intrinsics != BUNDLE_NO_INTRINSICS) + UnpackIntrinsicsFromArray(ceres_intrinsics, intrinsics); + + LG << "Final intrinsics: " << *intrinsics; + + if (evaluation) { + MultiviewBundlerPerformEvaluation(tracks, reconstruction, &all_cameras_R_t, + &problem, evaluation); + } + + // Separate step to adjust positions of tracks which are + // constant zero-weighted. + vector<Marker> zero_weight_markers; + for (int track = 0; track < tracks.MaxTrack(); ++track) { + if (zero_weight_tracks_flags[track]) { + vector<Marker> current_markers; + tracks.GetMarkersForTrack(track, ¤t_markers); + zero_weight_markers.reserve(zero_weight_markers.size() + + current_markers.size()); + for (int i = 0; i < current_markers.size(); ++i) { + zero_weight_markers.push_back(current_markers[i]); + } + } + } + + // zero-weight markers doesn't contribute to the bundle of pose + if (zero_weight_markers.size()) { + LG << "Refining position of constant zero-weighted tracks"; + EuclideanBundlePointsOnly(intrinsics->GetDistortionModelType(), + zero_weight_markers, + all_cameras_R_t, + ceres_intrinsics, + reconstruction, + camera_pose_map); + } +} + +bool EuclideanBundleAll(const Tracks &tracks, + Reconstruction *reconstruction) { + libmv::PolynomialCameraIntrinsics empty_intrinsics; + EuclideanBundleCommonIntrinsics(tracks, + BUNDLE_NO_INTRINSICS, + BUNDLE_NO_CONSTRAINTS, + reconstruction, + &empty_intrinsics, + NULL); + return true; +} + +} // namespace mv diff --git a/intern/libmv/libmv/autotrack/bundle.h b/intern/libmv/libmv/autotrack/bundle.h new file mode 100644 index 00000000000..c367802cf8f --- /dev/null +++ b/intern/libmv/libmv/autotrack/bundle.h @@ -0,0 +1,137 @@ +// Copyright (c) 2011, 2016 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: Tianwei Shen <shentianweipku@gmail.com> +// +// This is a autotrack equivalent bundle set, adapted from simple_pipeline, +// which replaces libmv with mv, includeing tracks and markers + +#ifndef LIBMV_AUTOTRACK_BUNDLE_H +#define LIBMV_AUTOTRACK_BUNDLE_H + +#include "libmv/numeric/numeric.h" +#include "libmv/autotrack/tracks.h" +#include "libmv/autotrack/reconstruction.h" +#include "libmv/simple_pipeline/callbacks.h" +#include "libmv/simple_pipeline/camera_intrinsics.h" + +using libmv::Mat; +using libmv::CameraIntrinsics; +using mv::Tracks; +using mv::Reconstruction; + +namespace mv { + +class EuclideanReconstruction; +class ProjectiveReconstruction; + +struct BundleEvaluation { + BundleEvaluation() : + num_cameras(0), + num_points(0), + evaluate_jacobian(false) { + } + + int num_cameras; /* Number of cameras appeared in bundle adjustment problem */ + int num_points; /* Number of points appeared in bundle adjustment problem */ + + /* When set to truth, jacobian of the problem after optimization will + * be evaluated and stored in \parameter jacobian */ + bool evaluate_jacobian; + + // Contains evaluated jacobian of the problem. + // Parameters are ordered in the following way: + // - Intrinsics block + // - Cameras (for each camera rotation goes first, then translation) + // - Points + Mat jacobian; +}; + +/*! + Refine camera poses and 3D coordinates using bundle adjustment. + + This routine adjusts all cameras positions, points, and the camera + intrinsics (assumed common across all images) in \a *reconstruction. This + assumes a full observation for reconstructed tracks; this implies that if + there is a reconstructed 3D point (a bundle) for a track, then all markers + for that track will be included in the minimization. \a tracks should + contain markers used in the initial reconstruction. + + The cameras, bundles, and intrinsics are refined in-place. + + Constraints denotes which blocks to keep constant during bundling. + For example it is useful to keep camera translations constant + when bundling tripod motions. + + If evaluaiton is not null, different evaluation statistics is filled in + there, plus all the requested additional information (like jacobian) is + also calculating there. Also see comments for BundleEvaluation. + + \note This assumes an outlier-free set of markers. + + \sa EuclideanResect, EuclideanIntersect, EuclideanReconstructTwoFrames +*/ +enum BundleIntrinsics { + BUNDLE_NO_INTRINSICS = 0, + BUNDLE_FOCAL_LENGTH = 1, + BUNDLE_PRINCIPAL_POINT = 2, + BUNDLE_RADIAL_K1 = 4, + BUNDLE_RADIAL_K2 = 8, + BUNDLE_RADIAL = 12, + BUNDLE_TANGENTIAL_P1 = 16, + BUNDLE_TANGENTIAL_P2 = 32, + BUNDLE_TANGENTIAL = 48, +}; +enum BundleConstraints { + BUNDLE_NO_CONSTRAINTS = 0, + BUNDLE_NO_TRANSLATION = 1, +}; + +void EuclideanBundleCommonIntrinsics( + const Tracks &tracks, + const int bundle_intrinsics, + const int bundle_constraints, + Reconstruction *reconstruction, + CameraIntrinsics *intrinsics, + BundleEvaluation *evaluation = NULL); + +/*! Refine all camera poses and 3D coordinates from all clips using bundle adjustment. + This is a renewed version for autotrack, adapted from libmv/simple_pipeline + + This routine adjusts all cameras and points in \a *reconstruction. This + assumes a full observation for reconstructed tracks; this implies that if + there is a reconstructed 3D point (a bundle) for a track, then all markers + for that track will be included in the minimization. \a tracks should + contain markers used in the initial reconstruction. + + The cameras and bundles (3D points) are refined in-place. + + \note This assumes an outlier-free set of markers. + \note This assumes a calibrated reconstruction, e.g. the markers are + already corrected for camera intrinsics and radial distortion. + + \sa EuclideanResect, EuclideanIntersect, EuclideanReconstructTwoFrames +*/ +bool EuclideanBundleAll(const Tracks &tracks, + Reconstruction *reconstruction); + +} // namespace mv + +#endif // LIBMV_AUTOTRACK_BUNDLE_H diff --git a/intern/libmv/libmv/autotrack/intersect.cc b/intern/libmv/libmv/autotrack/intersect.cc new file mode 100644 index 00000000000..67649b53437 --- /dev/null +++ b/intern/libmv/libmv/autotrack/intersect.cc @@ -0,0 +1,182 @@ +// Copyright (c) 2016 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: Tianwei Shen <shentianweipku@gmail.com> +// adapted from simple_pipeline/intersect.cc + +#include "libmv/autotrack/intersect.h" + +#include "libmv/base/vector.h" +#include "libmv/logging/logging.h" +#include "libmv/multiview/projection.h" +#include "libmv/multiview/triangulation.h" +#include "libmv/multiview/nviewtriangulation.h" +#include "libmv/numeric/numeric.h" +#include "libmv/numeric/levenberg_marquardt.h" +#include "libmv/autotrack/reconstruction.h" +#include "libmv/autotrack/tracks.h" + +#include "ceres/ceres.h" + +using libmv::Mat; +using libmv::Mat34; +using libmv::Mat2X; +using libmv::Vec4; + +namespace mv { + +namespace { + +class EuclideanIntersectCostFunctor { + public: + EuclideanIntersectCostFunctor(const Marker &marker, + const CameraPose &camera) + : marker_(marker), camera_(camera) {} + + template<typename T> + bool operator()(const T *X, T *residuals) const { + typedef Eigen::Matrix<T, 3, 3> Mat3; + typedef Eigen::Matrix<T, 3, 1> Vec3; + + Vec3 x(X); + Mat3 R(camera_.R.cast<T>()); + Vec3 t(camera_.t.cast<T>()); + + Vec3 projected = R * x + t; + projected /= projected(2); + + residuals[0] = (T(projected(0)) - T(marker_.center[0])) * T(marker_.weight); + residuals[1] = (T(projected(1)) - T(marker_.center[1])) * T(marker_.weight); + + return true; + } + + const Marker &marker_; + const CameraPose &camera_; +}; + +} // namespace + +bool EuclideanIntersect(const vector<Marker> &markers, + Reconstruction *reconstruction) { + if (markers.size() < 2) { + return false; + } + + // Compute projective camera matrices for the cameras the intersection is + // going to use. + Mat3 K = Mat3::Identity(); + vector<Mat34> cameras; + Mat34 P; + for (int i = 0; i < markers.size(); ++i) { + LG << "marker clip and frame: " << markers[i].clip << " " << markers[i].frame << std::endl; + CameraPose *camera = reconstruction->CameraPoseForFrame(markers[i].clip, markers[i].frame); + libmv::P_From_KRt(K, camera->R, camera->t, &P); + cameras.push_back(P); + } + + // Stack the 2D coordinates together as required by NViewTriangulate. + Mat2X points(2, markers.size()); + for (int i = 0; i < markers.size(); ++i) { + points(0, i) = markers[i].center[0]; + points(1, i) = markers[i].center[1]; + } + + Vec4 Xp; + LG << "Intersecting with " << markers.size() << " markers.\n"; + libmv::NViewTriangulateAlgebraic(points, cameras, &Xp); + + // Get euclidean version of the homogeneous point. + Xp /= Xp(3); + Vec3 X = Xp.head<3>(); + + ceres::Problem problem; + + // Add residual blocks to the problem. + int num_residuals = 0; + for (int i = 0; i < markers.size(); ++i) { + const Marker &marker = markers[i]; + if (marker.weight != 0.0) { + const CameraPose &camera = + *reconstruction->CameraPoseForFrame(marker.clip, marker.frame); + + problem.AddResidualBlock( + new ceres::AutoDiffCostFunction< + EuclideanIntersectCostFunctor, + 2, /* num_residuals */ + 3>(new EuclideanIntersectCostFunctor(marker, camera)), + NULL, + &X(0)); + num_residuals++; + } + } + + // TODO(sergey): Once we'll update Ceres to the next version + // we wouldn't need this check anymore -- Ceres will deal with + // zero-sized problems nicely. + LG << "Number of residuals: " << num_residuals << "\n"; + if (!num_residuals) { + LG << "Skipping running minimizer with zero residuals\n"; + + // We still add 3D point for the track regardless it was + // optimized or not. If track is a constant zero it'll use + // algebraic intersection result as a 3D coordinate. + + Vec3 point = X.head<3>(); + Point mv_point(markers[0].track, point); + reconstruction->AddPoint(mv_point); + + return true; + } + + // Configure the solve. + ceres::Solver::Options solver_options; + solver_options.linear_solver_type = ceres::DENSE_QR; + solver_options.max_num_iterations = 50; + solver_options.update_state_every_iteration = true; + solver_options.parameter_tolerance = 1e-16; + solver_options.function_tolerance = 1e-16; + + // Run the solve. + ceres::Solver::Summary summary; + ceres::Solve(solver_options, &problem, &summary); + + VLOG(1) << "Summary:\n" << summary.FullReport(); + + // Try projecting the point; make sure it's in front of everyone. + for (int i = 0; i < cameras.size(); ++i) { + const CameraPose &camera = + *reconstruction->CameraPoseForFrame(markers[i].clip, markers[i].frame); + Vec3 x = camera.R * X + camera.t; + if (x(2) < 0) { + //LOG(ERROR) << "POINT BEHIND CAMERA " << markers[i].image << ": " << x.transpose(); + return false; + } + } + + Vec3 point = X.head<3>(); + Point mv_point(markers[0].track, point); + reconstruction->AddPoint(mv_point); + + // TODO(keir): Add proper error checking. + return true; +} + +} // namespace mv diff --git a/intern/libmv/libmv/autotrack/intersect.h b/intern/libmv/libmv/autotrack/intersect.h new file mode 100644 index 00000000000..2c64a917e49 --- /dev/null +++ b/intern/libmv/libmv/autotrack/intersect.h @@ -0,0 +1,79 @@ +// Copyright (c) 2016 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: Tianwei Shen <shentianweipku@gmail.com> + +#ifndef LIBMV_AUTOTRACK_INTERSECT_H +#define LIBMV_AUTOTRACK_INTERSECT_H + +#include "libmv/base/vector.h" +#include "libmv/autotrack/tracks.h" +#include "libmv/autotrack/reconstruction.h" + +namespace mv { + +/*! + Estimate the 3D coordinates of a track by intersecting rays from images. + + This takes a set of markers, where each marker is for the same track but + different images, and reconstructs the 3D position of that track. Each of + the frames for which there is a marker for that track must have a + corresponding reconstructed camera in \a *reconstruction. + + \a markers should contain all \l Marker markers \endlink belonging to + tracks visible in all frames. + \a reconstruction should contain the cameras for all frames. + The new \l Point points \endlink will be inserted in \a reconstruction. + + \note This assumes a calibrated reconstruction, e.g. the markers are + already corrected for camera intrinsics and radial distortion. + \note This assumes an outlier-free set of markers. + + \sa EuclideanResect +*/ +bool EuclideanIntersect(const vector<Marker> &markers, + Reconstruction *reconstruction); + +/*! + Estimate the homogeneous coordinates of a track by intersecting rays. + + This takes a set of markers, where each marker is for the same track but + different images, and reconstructs the homogeneous 3D position of that + track. Each of the frames for which there is a marker for that track must + have a corresponding reconstructed camera in \a *reconstruction. + + \a markers should contain all \l Marker markers \endlink belonging to + tracks visible in all frames. + \a reconstruction should contain the cameras for all frames. + The new \l Point points \endlink will be inserted in \a reconstruction. + + \note This assumes that radial distortion is already corrected for, but + does not assume that e.g. focal length and principal point are + accounted for. + \note This assumes an outlier-free set of markers. + + \sa Resect +*/ +//bool ProjectiveIntersect(const vector<Marker> &markers, +// ProjectiveReconstruction *reconstruction); + +} // namespace mv + +#endif // LIBMV_AUTOTRACK_INTERSECT_H diff --git a/intern/libmv/libmv/autotrack/keyframe_selection.cc b/intern/libmv/libmv/autotrack/keyframe_selection.cc new file mode 100644 index 00000000000..cb8f5bde843 --- /dev/null +++ b/intern/libmv/libmv/autotrack/keyframe_selection.cc @@ -0,0 +1,455 @@ +// Copyright (c) 2016 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: Tianwei Shen <shentianweipku@gmail.com> +// adapted from simple_pipeline/keyframe_selection.cc + +#include "libmv/autotrack/keyframe_selection.h" + +#include "libmv/numeric/numeric.h" +#include "ceres/ceres.h" +#include "libmv/logging/logging.h" +#include "libmv/multiview/homography.h" +#include "libmv/multiview/fundamental.h" +#include "libmv/autotrack/intersect.h" +#include "libmv/autotrack/bundle.h" + +#include <Eigen/Eigenvalues> + +namespace mv { +using libmv::Vec; + +namespace { + +Mat3 IntrinsicsNormalizationMatrix(const CameraIntrinsics &intrinsics) { + Mat3 T = Mat3::Identity(), S = Mat3::Identity(); + + T(0, 2) = -intrinsics.principal_point_x(); + T(1, 2) = -intrinsics.principal_point_y(); + + S(0, 0) /= intrinsics.focal_length_x(); + S(1, 1) /= intrinsics.focal_length_y(); + + return S * T; +} + +// P.H.S. Torr +// Geometric Motion Segmentation and Model Selection +// +// http://reference.kfupm.edu.sa/content/g/e/geometric_motion_segmentation_and_model__126445.pdf +// +// d is the number of dimensions modeled +// (d = 3 for a fundamental matrix or 2 for a homography) +// k is the number of degrees of freedom in the model +// (k = 7 for a fundamental matrix or 8 for a homography) +// r is the dimension of the data +// (r = 4 for 2D correspondences between two frames) +double GRIC(const Vec &e, int d, int k, int r) { + int n = e.rows(); + double lambda1 = log(static_cast<double>(r)); + double lambda2 = log(static_cast<double>(r * n)); + + // lambda3 limits the residual error, and this paper + // http://elvera.nue.tu-berlin.de/files/0990Knorr2006.pdf + // suggests using lambda3 of 2 + // same value is used in Torr's Problem of degeneracy in structure + // and motion recovery from uncalibrated image sequences + // http://www.robots.ox.ac.uk/~vgg/publications/papers/torr99.ps.gz + double lambda3 = 2.0; + + // Variance of tracker position. Physically, this is typically about 0.1px, + // and when squared becomes 0.01 px^2. + double sigma2 = 0.01; + + // Finally, calculate the GRIC score. + double gric = 0.0; + for (int i = 0; i < n; i++) { + gric += std::min(e(i) * e(i) / sigma2, lambda3 * (r - d)); + } + gric += lambda1 * d * n; + gric += lambda2 * k; + return gric; +} + +// Compute a generalized inverse using eigen value decomposition, clamping the +// smallest eigenvalues if requested. This is needed to compute the variance of +// reconstructed 3D points. +// +// TODO(keir): Consider moving this into the numeric code, since this is not +// related to keyframe selection. +Mat PseudoInverseWithClampedEigenvalues(const Mat &matrix, + int num_eigenvalues_to_clamp) { + Eigen::EigenSolver<Mat> eigen_solver(matrix); + Mat D = eigen_solver.pseudoEigenvalueMatrix(); + Mat V = eigen_solver.pseudoEigenvectors(); + + // Clamp too-small singular values to zero to prevent numeric blowup. + double epsilon = std::numeric_limits<double>::epsilon(); + for (int i = 0; i < D.cols(); ++i) { + if (D(i, i) > epsilon) { + D(i, i) = 1.0 / D(i, i); + } else { + D(i, i) = 0.0; + } + } + + // Apply the clamp. + for (int i = D.cols() - num_eigenvalues_to_clamp; i < D.cols(); ++i) { + D(i, i) = 0.0; + } + return V * D * V.inverse(); +} + +void FilterZeroWeightMarkersFromTracks(const Tracks &tracks, + Tracks *filtered_tracks) { + vector<Marker> all_markers; + tracks.GetAllMarkers(&all_markers); + + for (int i = 0; i < all_markers.size(); ++i) { + Marker &marker = all_markers[i]; + if (marker.weight != 0.0) { + filtered_tracks->AddMarker(marker); + } + } +} + +} // namespace + +void SelectClipKeyframesBasedOnGRICAndVariance(const int clip_index, + const Tracks &_tracks, + const CameraIntrinsics &intrinsics, + vector<int> &keyframes) { + // Mirza Tahir Ahmed, Matthew N. Dailey + // Robust key frame extraction for 3D reconstruction from video streams + // + // http://www.cs.ait.ac.th/~mdailey/papers/Tahir-KeyFrame.pdf + + Tracks filtered_tracks; + FilterZeroWeightMarkersFromTracks(_tracks, &filtered_tracks); + + int max_frame = filtered_tracks.MaxFrame(clip_index); + int next_keyframe = 1; + int number_keyframes = 0; + + // Limit correspondence ratio from both sides. + // On the one hand if number of correspondent features is too low, + // triangulation will suffer. + // On the other hand high correspondence likely means short baseline. + // which also will affect accuracy. + const double Tmin = 0.8; + const double Tmax = 1.0; + + Mat3 N = IntrinsicsNormalizationMatrix(intrinsics); + Mat3 N_inverse = N.inverse(); + + double Sc_best = std::numeric_limits<double>::max(); + double success_intersects_factor_best = 0.0f; + + while (next_keyframe != -1) { + int current_keyframe = next_keyframe; + double Sc_best_candidate = std::numeric_limits<double>::max(); + + LG << "Found keyframe " << next_keyframe; + + number_keyframes++; + next_keyframe = -1; + + for (int candidate_image = current_keyframe + 1; + candidate_image <= max_frame; + candidate_image++) { + // Conjunction of all markers from both keyframes + vector<Marker> all_markers; + filtered_tracks.GetMarkersInBothFrames(clip_index, current_keyframe, + clip_index, candidate_image, &all_markers); + + // Match keypoints between frames current_keyframe and candidate_image + vector<Marker> tracked_markers; + filtered_tracks.GetMarkersForTracksInBothFrames(clip_index, current_keyframe, + clip_index, candidate_image, &tracked_markers); + + // Correspondences in normalized space + Mat x1, x2; + CoordinatesForMarkersInFrame(tracked_markers, clip_index, current_keyframe, &x1); + CoordinatesForMarkersInFrame(tracked_markers, clip_index, candidate_image, &x2); + + LG << "Found " << x1.cols() + << " correspondences between " << current_keyframe + << " and " << candidate_image; + + // Not enough points to construct fundamental matrix + if (x1.cols() < 8 || x2.cols() < 8) + continue; + + // STEP 1: Correspondence ratio constraint + int Tc = tracked_markers.size(); + int Tf = all_markers.size(); + double Rc = static_cast<double>(Tc) / Tf; + + LG << "Correspondence between " << current_keyframe + << " and " << candidate_image + << ": " << Rc; + + if (Rc < Tmin || Rc > Tmax) + continue; + + Mat3 H, F; + + // Estimate homography using default options. + libmv::EstimateHomographyOptions estimate_homography_options; + libmv::EstimateHomography2DFromCorrespondences(x1, + x2, + estimate_homography_options, + &H); + + // Convert homography to original pixel space. + H = N_inverse * H * N; + + libmv::EstimateFundamentalOptions estimate_fundamental_options; + libmv::EstimateFundamentalFromCorrespondences(x1, + x2, + estimate_fundamental_options, + &F); + + // Convert fundamental to original pixel space. + F = N_inverse * F * N; + + // TODO(sergey): STEP 2: Discard outlier matches + + // STEP 3: Geometric Robust Information Criteria + + // Compute error values for homography and fundamental matrices + Vec H_e, F_e; + H_e.resize(x1.cols()); + F_e.resize(x1.cols()); + for (int i = 0; i < x1.cols(); i++) { + libmv::Vec2 current_x1, current_x2; + + intrinsics.NormalizedToImageSpace(x1(0, i), x1(1, i), + ¤t_x1(0), ¤t_x1(1)); + + intrinsics.NormalizedToImageSpace(x2(0, i), x2(1, i), + ¤t_x2(0), ¤t_x2(1)); + + H_e(i) = libmv::SymmetricGeometricDistance(H, current_x1, current_x2); + F_e(i) = libmv::SymmetricEpipolarDistance(F, current_x1, current_x2); + } + + LG << "H_e: " << H_e.transpose(); + LG << "F_e: " << F_e.transpose(); + + // Degeneracy constraint + double GRIC_H = GRIC(H_e, 2, 8, 4); + double GRIC_F = GRIC(F_e, 3, 7, 4); + + LG << "GRIC values for frames " << current_keyframe + << " and " << candidate_image + << ", H-GRIC: " << GRIC_H + << ", F-GRIC: " << GRIC_F; + + if (GRIC_H <= GRIC_F) { + LG << "Skip " << candidate_image << " since GRIC_H <= GRIC_F\n"; + continue; + } + + // TODO(sergey): STEP 4: PELC criterion + + // STEP 5: Estimation of reconstruction error + // + // Uses paper Keyframe Selection for Camera Motion and Structure + // Estimation from Multiple Views + // Uses ftp://ftp.tnt.uni-hannover.de/pub/papers/2004/ECCV2004-TTHBAW.pdf + // Basically, equation (15) + // + // TODO(sergey): separate all the constraints into functions, + // this one is getting to much cluttered already + + // Definitions in equation (15): + // - I is the number of 3D feature points + // - A is the number of essential parameters of one camera + + Reconstruction reconstruction; + + // The F matrix should be an E matrix, but squash it just to be sure + + // Reconstruction should happen using normalized fundamental matrix + Mat3 F_normal = N * F * N_inverse; + + Mat3 E; + libmv::FundamentalToEssential(F_normal, &E); + + // Recover motion between the two images. Since this function assumes a + // calibrated camera, use the identity for K + Mat3 R; + Vec3 t; + Mat3 K = Mat3::Identity(); + + if (!libmv::MotionFromEssentialAndCorrespondence(E, + K, x1.col(0), + K, x2.col(0), + &R, &t)) { + LG << "Failed to compute R and t from E and K"; + continue; + } + + LG << "Camera transform between frames " << current_keyframe + << " and " << candidate_image + << ":\nR:\n" << R + << "\nt:" << t.transpose(); + + // First camera is identity, second one is relative to it + CameraPose current_keyframe_pose(clip_index, current_keyframe, 0, Mat3::Identity(), Vec3::Zero()); + reconstruction.AddCameraPose(current_keyframe_pose); + CameraPose candidate_keyframe_pose(clip_index, candidate_image, 0, R, t); + reconstruction.AddCameraPose(candidate_keyframe_pose); + + // Reconstruct 3D points + int intersects_total = 0, intersects_success = 0; + for (int i = 0; i < tracked_markers.size(); i++) { + if (!reconstruction.PointForTrack(tracked_markers[i].track)) { + vector<Marker> reconstructed_markers; + + int track = tracked_markers[i].track; + + reconstructed_markers.push_back(tracked_markers[i]); + + // We know there're always only two markers for a track + // Also, we're using brute-force search because we don't + // actually know about markers layout in a list, but + // at this moment this cycle will run just once, which + // is not so big deal + + for (int j = i + 1; j < tracked_markers.size(); j++) { + if (tracked_markers[j].track == track) { + reconstructed_markers.push_back(tracked_markers[j]); + break; + } + } + + intersects_total++; + + if (EuclideanIntersect(reconstructed_markers, &reconstruction)) { + LG << "Ran Intersect() for track " << track; + intersects_success++; + } else { + LG << "Failed to intersect track " << track; + } + } + } + + double success_intersects_factor = + (double) intersects_success / intersects_total; + + if (success_intersects_factor < success_intersects_factor_best) { + LG << "Skip keyframe candidate because of " + "lower successful intersections ratio"; + + continue; + } + + success_intersects_factor_best = success_intersects_factor; + + Tracks two_frames_tracks(tracked_markers); + libmv::PolynomialCameraIntrinsics empty_intrinsics; + BundleEvaluation evaluation; + evaluation.evaluate_jacobian = true; + + EuclideanBundleCommonIntrinsics(two_frames_tracks, + BUNDLE_NO_INTRINSICS, + BUNDLE_NO_CONSTRAINTS, + &reconstruction, + &empty_intrinsics, + &evaluation); + + Mat &jacobian = evaluation.jacobian; + + Mat JT_J = jacobian.transpose() * jacobian; + // There are 7 degrees of freedom, so clamp them out. + Mat JT_J_inv = PseudoInverseWithClampedEigenvalues(JT_J, 7); + + Mat temp_derived = JT_J * JT_J_inv * JT_J; + bool is_inversed = (temp_derived - JT_J).cwiseAbs2().sum() < + 1e-4 * std::min(temp_derived.cwiseAbs2().sum(), + JT_J.cwiseAbs2().sum()); + + LG << "Check on inversed: " << (is_inversed ? "true" : "false" ) + << ", det(JT_J): " << JT_J.determinant(); + + if (!is_inversed) { + LG << "Ignoring candidature due to poor jacobian stability"; + continue; + } + + Mat Sigma_P; + Sigma_P = JT_J_inv.bottomRightCorner(evaluation.num_points * 3, + evaluation.num_points * 3); + + int I = evaluation.num_points; + int A = 12; + + double Sc = static_cast<double>(I + A) / libmv::Square(3 * I) * Sigma_P.trace(); + + LG << "Expected estimation error between " + << current_keyframe << " and " + << candidate_image << ": " << Sc; + + // Pairing with a lower Sc indicates a better choice + if (Sc > Sc_best_candidate) + continue; + + Sc_best_candidate = Sc; + + next_keyframe = candidate_image; + } + + // This is a bit arbitrary and main reason of having this is to deal + // better with situations when there's no keyframes were found for + // current keyframe this could happen when there's no so much parallax + // in the beginning of image sequence and then most of features are + // getting occluded. In this case there could be good keyframe pair in + // the middle of the sequence + // + // However, it's just quick hack and smarter way to do this would be nice + if (next_keyframe == -1) { + next_keyframe = current_keyframe + 10; + number_keyframes = 0; + + if (next_keyframe >= max_frame) + break; + + LG << "Starting searching for keyframes starting from " << next_keyframe; + } else { + // New pair's expected reconstruction error is lower + // than existing pair's one. + // + // For now let's store just one candidate, easy to + // store more candidates but needs some thoughts + // how to choose best one automatically from them + // (or allow user to choose pair manually). + if (Sc_best > Sc_best_candidate) { + keyframes.clear(); + keyframes.push_back(current_keyframe); + keyframes.push_back(next_keyframe); + Sc_best = Sc_best_candidate; + } + } + } +} + +} // namespace mv diff --git a/intern/libmv/libmv/autotrack/keyframe_selection.h b/intern/libmv/libmv/autotrack/keyframe_selection.h new file mode 100644 index 00000000000..a77d5b30775 --- /dev/null +++ b/intern/libmv/libmv/autotrack/keyframe_selection.h @@ -0,0 +1,58 @@ +// Copyright (c) 2016 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: Tianwei Shen <shentianweipku@gmail.com> +// adapted from simple_pipeline/keyframe_selection.h + +#ifndef LIBMV_AUTOTRACK_KEYFRAME_SELECTION_H_ +#define LIBMV_AUTOTRACK_KEYFRAME_SELECTION_H_ + +#include "libmv/base/vector.h" +#include "libmv/autotrack/tracks.h" +#include "libmv/simple_pipeline/camera_intrinsics.h" + +namespace mv { + +// Get list of all images which are good enough to be as keyframes in a clip +// from camera reconstruction. Based on GRIC criteria and uses Pollefeys' +// approach for correspondence ratio constraint. +// +// As an additional, additional criteria based on reconstruction +// variance is used. This means if correspondence and GRIC criteria +// are passed, two-frames reconstruction using candidate keyframes +// happens. After reconstruction variance of 3D points is calculating +// and if expected error estimation is too large, keyframe candidate +// is rejecting. +// +// \param clip_index is the clip index to extract keyframes from +// \param tracks contains all tracked correspondences between frames +// expected to be undistorted and normalized +// \param intrinsics is camera intrinsics +// \param keyframes will contain all images number which are considered +// good to be used for reconstruction +void SelectClipKeyframesBasedOnGRICAndVariance( + const int clip_index, + const Tracks &tracks, + const libmv::CameraIntrinsics &intrinsics, + vector<int> &keyframes); + +} // namespace mv + +#endif // LIBMV_AUTOTRACK_KEYFRAME_SELECTION_H_ diff --git a/intern/libmv/libmv/autotrack/pipeline.cc b/intern/libmv/libmv/autotrack/pipeline.cc new file mode 100644 index 00000000000..ec85cc78e1f --- /dev/null +++ b/intern/libmv/libmv/autotrack/pipeline.cc @@ -0,0 +1,406 @@ +// Copyright (c) 2016 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// Author: Tianwei Shen <shentianweipku@gmail.com> +// This file contains the multi-view reconstruction pipeline, such as camera resection. + +#include "libmv/autotrack/pipeline.h" + +#include <cstdio> + +#include "libmv/logging/logging.h" +#include "libmv/autotrack/bundle.h" +#include "libmv/autotrack/reconstruction.h" +#include "libmv/autotrack/tracks.h" +#include "libmv/autotrack/intersect.h" +#include "libmv/autotrack/resect.h" +#include "libmv/simple_pipeline/camera_intrinsics.h" + +#ifdef _MSC_VER +# define snprintf _snprintf +#endif + +namespace mv { +namespace { + +// Use this functor-like struct to reuse reconstruction pipeline code +// in the future, in case we will do projective reconstruction +struct EuclideanPipelineRoutines { + typedef ::mv::Reconstruction Reconstruction; + typedef CameraPose Camera; + typedef ::mv::Point Point; + + static void Bundle(const Tracks &tracks, + Reconstruction *reconstruction) { + EuclideanBundleAll(tracks, reconstruction); + } + + static bool Resect(const vector<Marker> &markers, + Reconstruction *reconstruction, + bool final_pass, + int intrinsics_index) { + return EuclideanResect(markers, reconstruction, final_pass, intrinsics_index); + } + + static bool Intersect(const vector<Marker> &markers, + Reconstruction *reconstruction) { + return EuclideanIntersect(markers, reconstruction); + } + + static Marker ProjectMarker(const Point &point, + const CameraPose &camera, + const CameraIntrinsics &intrinsics) { + Vec3 projected = camera.R * point.X + camera.t; + projected /= projected(2); + + Marker reprojected_marker; + double normalized_x, normalized_y; + intrinsics.ApplyIntrinsics(projected(0), + projected(1), + &normalized_x, + &normalized_y); + reprojected_marker.center[0] = normalized_x; + reprojected_marker.center[1] = normalized_y; + + reprojected_marker.clip = camera.clip; + reprojected_marker.frame = camera.frame; + reprojected_marker.track = point.track; + return reprojected_marker; + } +}; + +} // namespace + +static void CompleteReconstructionLogProgress( + libmv::ProgressUpdateCallback *update_callback, + double progress, + const char *step = NULL) { + if (update_callback) { + char message[256]; + + if (step) + snprintf(message, sizeof(message), "Completing solution %d%% | %s", + (int)(progress*100), step); + else + snprintf(message, sizeof(message), "Completing solution %d%%", + (int)(progress*100)); + + update_callback->invoke(progress, message); + } +} + +template<typename PipelineRoutines> +bool InternalCompleteReconstruction( + const Tracks &tracks, + typename PipelineRoutines::Reconstruction *reconstruction, + libmv::ProgressUpdateCallback *update_callback = NULL) { + int clip_num = tracks.MaxClip() + 1; + int num_frames = 0; + for(int i = 0; i < clip_num; i++) { + num_frames += tracks.MaxFrame(i) + 1; + } + + int max_track = tracks.MaxTrack(); + int num_resects = -1; + int num_intersects = -1; + int total_resects = 0; + LG << "Max track: " << max_track << "\n"; + LG << "Number of total frames: " << num_frames << "\n"; + LG << "Number of markers: " << tracks.NumMarkers() << "\n"; + while (num_resects != 0 || num_intersects != 0) { + // Do all possible intersections. + num_intersects = 0; + for (int track = 0; track <= max_track; ++track) { + if (reconstruction->PointForTrack(track)) { // track has already been added + LG << "Skipping point: " << track << "\n"; + continue; + } + vector<Marker> all_markers; + tracks.GetMarkersForTrack(track, &all_markers); + LG << "Got " << all_markers.size() << " markers for track " << track << "\n"; + + vector<Marker> reconstructed_markers; + for (int i = 0; i < all_markers.size(); ++i) { + if (reconstruction->CameraPoseForFrame(all_markers[i].clip, all_markers[i].frame)) { + reconstructed_markers.push_back(all_markers[i]); + } + } + LG << "Got " << reconstructed_markers.size() << " reconstructed markers for track " << track << "\n"; + if (reconstructed_markers.size() >= 2) { + CompleteReconstructionLogProgress(update_callback, + (double)total_resects/(num_frames)); + if (PipelineRoutines::Intersect(reconstructed_markers, + reconstruction)) { + num_intersects++; + LG << "Ran Intersect() for track " << track << "\n"; + } else { + LG << "Failed Intersect() for track " << track << "\n"; + } + } + } + // bundle the newly added points + if (num_intersects) { + CompleteReconstructionLogProgress(update_callback, + (double)total_resects/(num_frames), + "Bundling..."); + PipelineRoutines::Bundle(tracks, reconstruction); + LG << "Ran Bundle() after intersections."; + } + LG << "Did " << num_intersects << " intersects.\n"; + + // Do all possible resections. + num_resects = 0; + for(int clip = 0; clip < clip_num; clip++) { + const int max_image = tracks.MaxFrame(clip); + for (int image = 0; image <= max_image; ++image) { + if (reconstruction->CameraPoseForFrame(clip, image)) { // this camera pose has been added + LG << "Skipping frame: " << clip << " " << image << "\n"; + continue; + } + vector<Marker> all_markers; + tracks.GetMarkersInFrame(clip, image, &all_markers); + LG << "Got " << all_markers.size() << " markers for frame " << clip << ", " << image << "\n"; + + vector<Marker> reconstructed_markers; + for (int i = 0; i < all_markers.size(); ++i) { + if (reconstruction->PointForTrack(all_markers[i].track)) { // 3d points have been added + reconstructed_markers.push_back(all_markers[i]); + } + } + LG << "Got " << reconstructed_markers.size() << " reconstructed markers for frame " + << clip << " " << image << "\n"; + if (reconstructed_markers.size() >= 5) { + CompleteReconstructionLogProgress(update_callback, + (double)total_resects/(num_frames)); + if (PipelineRoutines::Resect(reconstructed_markers, + reconstruction, false, + reconstruction->GetIntrinsicsMap(clip, image))) { + num_resects++; + total_resects++; + LG << "Ran Resect() for frame (" << clip << ", " << image << ")\n"; + } else { + LG << "Failed Resect() for frame (" << clip << ", " << image << ")\n"; + } + } + } + } + if (num_resects) { + CompleteReconstructionLogProgress(update_callback, + (double)total_resects/(num_frames), + "Bundling..."); + PipelineRoutines::Bundle(tracks, reconstruction); + } + LG << "Did " << num_resects << " resects.\n"; + } + + // One last pass... + LG << "[InternalCompleteReconstruction] Ran last pass\n"; + num_resects = 0; + for(int clip = 0; clip < clip_num; clip++) { + int max_image = tracks.MaxFrame(clip); + for (int image = 0; image <= max_image; ++image) { + if (reconstruction->CameraPoseForFrame(clip, image)) { + LG << "Skipping frame: " << clip << " " << image << "\n"; + continue; + } + vector<Marker> all_markers; + tracks.GetMarkersInFrame(clip, image, &all_markers); + + vector<Marker> reconstructed_markers; + for (int i = 0; i < all_markers.size(); ++i) { + if (reconstruction->PointForTrack(all_markers[i].track)) { + reconstructed_markers.push_back(all_markers[i]); + } + } + if (reconstructed_markers.size() >= 5) { + CompleteReconstructionLogProgress(update_callback, + (double)total_resects/(max_image)); + if (PipelineRoutines::Resect(reconstructed_markers, + reconstruction, true, + reconstruction->GetIntrinsicsMap(clip, image))) { + num_resects++; + LG << "Ran final Resect() for image " << image; + } else { + LG << "Failed final Resect() for image " << image; + } + } + } + } + if (num_resects) { + CompleteReconstructionLogProgress(update_callback, + (double)total_resects/(num_frames), + "Bundling..."); + PipelineRoutines::Bundle(tracks, reconstruction); + } + return true; +} + +template<typename PipelineRoutines> +double InternalReprojectionError( + const Tracks &image_tracks, + const typename PipelineRoutines::Reconstruction &reconstruction, + const CameraIntrinsics &intrinsics) { + int num_skipped = 0; + int num_reprojected = 0; + double total_error = 0.0; + vector<Marker> markers; + image_tracks.GetAllMarkers(&markers); + for (int i = 0; i < markers.size(); ++i) { + double weight = markers[i].weight; + const typename PipelineRoutines::Camera *camera = + reconstruction.CameraPoseForFrame(markers[i].clip, markers[i].frame); + const typename PipelineRoutines::Point *point = + reconstruction.PointForTrack(markers[i].track); + if (!camera || !point || weight == 0.0) { + num_skipped++; + continue; + } + num_reprojected++; + + Marker reprojected_marker = + PipelineRoutines::ProjectMarker(*point, *camera, intrinsics); + double ex = (reprojected_marker.center[0] - markers[i].center[0]) * weight; + double ey = (reprojected_marker.center[1] - markers[i].center[1]) * weight; + + const int N = 100; + char line[N]; + snprintf(line, N, + "frame (%d, %d) track %-3d " + "x %7.1f y %7.1f " + "rx %7.1f ry %7.1f " + "ex %7.1f ey %7.1f" + " e %7.1f", + markers[i].clip, + markers[i].frame, + markers[i].track, + markers[i].center[0], + markers[i].center[1], + reprojected_marker.center[0], + reprojected_marker.center[1], + ex, + ey, + sqrt(ex*ex + ey*ey)); + VLOG(1) << line; + total_error += sqrt(ex*ex + ey*ey); + } + LG << "Skipped " << num_skipped << " markers."; + LG << "Reprojected " << num_reprojected << " markers."; + LG << "Total error: " << total_error; + LG << "Average error: " << (total_error / num_reprojected) << " [pixels]."; + return total_error / num_reprojected; +} + +double EuclideanReprojectionError(const Tracks &tracks, + const Reconstruction &reconstruction, + const CameraIntrinsics &intrinsics) { + return InternalReprojectionError<EuclideanPipelineRoutines>(tracks, + reconstruction, + intrinsics); +} + +bool EuclideanCompleteMultiviewReconstruction(const Tracks &tracks, + Reconstruction *reconstruction, + libmv::ProgressUpdateCallback *update_callback) { + return InternalCompleteReconstruction<EuclideanPipelineRoutines>(tracks, + reconstruction, + update_callback); +} + +void InvertIntrinsicsForTracks(const Tracks &raw_tracks, + const CameraIntrinsics &camera_intrinsics, + Tracks *calibrated_tracks) { + vector<Marker> markers; + raw_tracks.GetAllMarkers(&markers); + for (int i = 0; i < markers.size(); ++i) { + double normalized_x, normalized_y; + camera_intrinsics.InvertIntrinsics(markers[i].center[0], markers[i].center[1], + &normalized_x, &normalized_y); + markers[i].center[0] = normalized_x, markers[i].center[1] = normalized_y; + } + *calibrated_tracks = Tracks(markers); +} + +void EuclideanScaleToUnity(Reconstruction *reconstruction) { + int clip_num = reconstruction->GetClipNum(); + const vector<vector<CameraPose> >& all_cameras = reconstruction->camera_poses(); + + LG << "[EuclideanScaleToUnity] camera clip number: " << clip_num << '\n'; + // Calculate center of the mass of all cameras. + int total_valid_cameras = 0; + Vec3 cameras_mass_center = Vec3::Zero(); + for (int i = 0; i < clip_num; i++) { + for (int j = 0; j < all_cameras[i].size(); ++j) { + if (all_cameras[i][j].clip >= 0) { // primary camera index is 0 + cameras_mass_center += all_cameras[i][j].t; + total_valid_cameras++; + } + } + } + if (total_valid_cameras == 0) { + LG << "[EuclideanScaleToUnity] no valid camera for rescaling\n"; + return; + } + + cameras_mass_center /= total_valid_cameras; + LG << "[EuclideanScaleToUnity] valid camera number: " << total_valid_cameras << '\n'; + + // Find the most distant camera from the mass center. + double max_distance = 0.0; + for(int i = 0; i < clip_num; i++) { + for (int j = 0; j < all_cameras[i].size(); ++j) { + double distance = (all_cameras[i][j].t - cameras_mass_center).squaredNorm(); + if (distance > max_distance) { + max_distance = distance; + } + } + } + + if (max_distance == 0.0) { + LG << "Cameras position variance is too small, can not rescale\n"; + return; + } + + double scale_factor = 1.0 / sqrt(max_distance); + LG << "rescale factor: " << scale_factor << "\n"; + + // Rescale cameras positions. + for(int i = 0; i < clip_num; i++) { + for (int j = 0; j < all_cameras[i].size(); ++j) { + int frame = all_cameras[i][j].frame; + CameraPose *camera = reconstruction->CameraPoseForFrame(i, frame); + if (camera != NULL) + camera->t = camera->t * scale_factor; + else + LG << "[EuclideanScaleToUnity] invalid camera: " << i << " " << frame << "\n"; + } + } + + // Rescale points positions. + vector<Point> all_points = reconstruction->AllPoints(); + for (int i = 0; i < all_points.size(); ++i) { + int track = all_points[i].track; + Point *point = reconstruction->PointForTrack(track); + if (point != NULL) + point->X = point->X * scale_factor; + else + LG << "[EuclideanScaleToUnity] invalid point: " << i << "\n"; + } +} + +} // namespace mv diff --git a/intern/libmv/libmv/autotrack/pipeline.h b/intern/libmv/libmv/autotrack/pipeline.h new file mode 100644 index 00000000000..6ab3266461c --- /dev/null +++ b/intern/libmv/libmv/autotrack/pipeline.h @@ -0,0 +1,71 @@ +// Copyright (c) 2016 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// Author: Tianwei Shen <shentianweipku@gmail.com> +// This file contains the multi-view reconstruction pipeline, such as camera resection. + +#ifndef LIBMV_AUTOTRACK_PIPELINE_H_ +#define LIBMV_AUTOTRACK_PIPELINE_H_ + +#include "libmv/autotrack/tracks.h" +#include "libmv/autotrack/reconstruction.h" +#include "libmv/simple_pipeline/callbacks.h" +#include "libmv/simple_pipeline/camera_intrinsics.h" + +using libmv::CameraIntrinsics; + +namespace mv { + +/*! + Estimate multi-view camera poses and scene 3D coordinates for all frames and tracks. + + This method should be used once there is an initial reconstruction in + place, for example by reconstructing from two frames that have a sufficient + baseline and number of tracks in common. This function iteratively + triangulates points that are visible by cameras that have their pose + estimated, then resections (i.e. estimates the pose) of cameras that are + not estimated yet that can see triangulated points. This process is + repeated until all points and cameras are estimated. Periodically, bundle + adjustment is run to ensure a quality reconstruction. + + \a tracks should contain markers used in the reconstruction. + \a reconstruction should contain at least some 3D points or some estimated + cameras. The minimum number of cameras is two (with no 3D points) and the + minimum number of 3D points (with no estimated cameras) is 5. + + \sa EuclideanResect, EuclideanIntersect, EuclideanBundle +*/ +bool EuclideanCompleteMultiviewReconstruction( + const Tracks &tracks, + Reconstruction *reconstruction, + libmv::ProgressUpdateCallback *update_callback = NULL); + +double EuclideanReprojectionError(const Tracks &image_tracks, + const Reconstruction &reconstruction, + const CameraIntrinsics &intrinsics); + +void InvertIntrinsicsForTracks(const Tracks &raw_tracks, + const CameraIntrinsics &camera_intrinsics, + Tracks *calibrated_tracks); + +void EuclideanScaleToUnity(Reconstruction *reconstruction); + +} // namespace mv + +#endif // LIBMV_AUTOTRACK_PIPELINE_H_ diff --git a/intern/libmv/libmv/autotrack/reconstruction.cc b/intern/libmv/libmv/autotrack/reconstruction.cc new file mode 100644 index 00000000000..3c1acf5dfff --- /dev/null +++ b/intern/libmv/libmv/autotrack/reconstruction.cc @@ -0,0 +1,247 @@ +// Copyright (c) 2016 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// +// Author: shentianweipku@gmail.com (Tianwei Shen) + +#include "libmv/autotrack/reconstruction.h" +#include "libmv/multiview/fundamental.h" +#include "libmv/autotrack/marker.h" +#include "libmv/autotrack/tracks.h" +#include "libmv/numeric/numeric.h" +#include "libmv/logging/logging.h" +#include "libmv/simple_pipeline/camera_intrinsics.h" + +using mv::Marker; +using mv::Tracks; +using libmv::Mat; +using libmv::Mat2; +using libmv::Mat3; +using libmv::Vec; +using libmv::Vec2; + +namespace mv { + +static void GetFramesInMarkers(const vector<Marker> &markers, + int *image1, int *image2) { + if (markers.size() < 2) { + return; + } + *image1 = markers[0].frame; + for (int i = 1; i < markers.size(); ++i) { + if (markers[i].frame != *image1) { + *image2 = markers[i].frame; + return; + } + } + *image2 = -1; + LOG(FATAL) << "Only one image in the markers."; +} + +/* markers come from two views in the same clip, + * reconstruction should be new and empty + */ +bool ReconstructTwoFrames(const vector<Marker> &markers, + const int clip, + Reconstruction *reconstruction) +{ + if (markers.size() < 16) { + LG << "Not enough markers to initialize from two frames: " << markers.size(); + return false; + } + + int frame1, frame2; + GetFramesInMarkers(markers, &frame1, &frame2); + + Mat x1, x2; + CoordinatesForMarkersInFrame(markers, clip, frame1, &x1); + CoordinatesForMarkersInFrame(markers, clip, frame2, &x2); + + Mat3 F; + libmv::NormalizedEightPointSolver(x1, x2, &F); + + // The F matrix should be an E matrix, but squash it just to be sure. + Mat3 E; + libmv::FundamentalToEssential(F, &E); + + // Recover motion between the two images. Since this function assumes a + // calibrated camera, use the identity for K. + Mat3 R; + Vec3 t; + Mat3 K = Mat3::Identity(); + if (!libmv::MotionFromEssentialAndCorrespondence(E, K, x1.col(0), K, x2.col(0), &R, &t)) { + LG << "Failed to compute R and t from E and K."; + return false; + } + + // frame 1 gets the reference frame, frame 2 gets the relative motion. + CameraPose pose1(clip, frame1, reconstruction->GetIntrinsicsMap(clip, frame1), Mat3::Identity(), Vec3::Zero()); + CameraPose pose2(clip, frame2, reconstruction->GetIntrinsicsMap(clip, frame2), R, t); + reconstruction->AddCameraPose(pose1); + reconstruction->AddCameraPose(pose2); + + LG << "From two frame reconstruction got:\nR:\n" << R << "\nt:" << t.transpose(); + return true; +} + +// ================== mv::Reconstruction implementation =================== +// push a new cameraIntrinsics and return the index +int Reconstruction::AddCameraIntrinsics(CameraIntrinsics *intrinsics_ptr) { + camera_intrinsics_.push_back(intrinsics_ptr); + return camera_intrinsics_.size()-1; +} + +void Reconstruction::AddCameraPose(const CameraPose& pose) { + if (camera_poses_.size() < pose.clip + 1) { + camera_poses_.resize(pose.clip+1); + } + if (camera_poses_[pose.clip].size() < pose.frame + 1) { + camera_poses_[pose.clip].resize(pose.frame+1); + } + // copy form pose to camera_poses_ + camera_poses_[pose.clip][pose.frame].clip = pose.clip; + camera_poses_[pose.clip][pose.frame].frame = pose.frame; + camera_poses_[pose.clip][pose.frame].intrinsics = pose.intrinsics; + camera_poses_[pose.clip][pose.frame].R = pose.R; + camera_poses_[pose.clip][pose.frame].t = pose.t; +} + +int Reconstruction::GetClipNum() const { + return camera_poses_.size(); +} + +int Reconstruction::GetAllPoseNum() const { + int all_pose = 0; + for (int i = 0; i < camera_poses_.size(); ++i) { + all_pose += camera_poses_[i].size(); + } + return all_pose; +} + +CameraPose* Reconstruction::CameraPoseForFrame(int clip, int frame) { + if (camera_poses_.size() <= clip) + return NULL; + if (camera_poses_[clip].size() <= frame) + return NULL; + if (camera_poses_[clip][frame].clip == -1) // this CameraPose is uninitilized + return NULL; + return &(camera_poses_[clip][frame]); +} + +const CameraPose* Reconstruction::CameraPoseForFrame(int clip, int frame) const { + if (camera_poses_.size() <= clip) + return NULL; + if (camera_poses_[clip].size() <= frame) + return NULL; + if (camera_poses_[clip][frame].clip == -1) // this CameraPose is uninitilized + return NULL; + return (const CameraPose*) &(camera_poses_[clip][frame]); +} + +Point* Reconstruction::PointForTrack(int track) { + if (track < 0 || track >= points_.size()) { + return NULL; + } + Point *point = &points_[track]; + if (point->track == -1) { + return NULL; + } + return point; +} + +const Point* Reconstruction::PointForTrack(int track) const { + if (track < 0 || track >= points_.size()) { + return NULL; + } + const Point *point = &points_[track]; + if (point->track == -1) { // initialized but not set, return NULL + return NULL; + } + return point; +} + +int Reconstruction::AddPoint(const Point& point) { + LG << "InsertPoint " << point.track << ":\n" << point.X; + if (point.track >= points_.size()) { + points_.resize(point.track + 1); + } + points_[point.track].track = point.track; + points_[point.track].X = point.X; + return point.track; +} + +const vector<vector<CameraPose> >& Reconstruction::camera_poses() const { + return camera_poses_; +} + +const vector<Point>& Reconstruction::AllPoints() const { + return points_; +} + +int Reconstruction::GetReconstructedCameraNum() const { + int reconstructed_num = 0; + for (int i = 0; i < camera_poses_.size(); i++) { + for (int j = 0; j < camera_poses_[i].size(); j++) { + if (camera_poses_[i][j].clip != -1 && camera_poses_[i][j].frame != -1) + reconstructed_num++; + } + } + return reconstructed_num; +} + +void Reconstruction::InitIntrinsicsMap(Tracks &tracks) { + int clip_num = tracks.MaxClip() + 1; + intrinsics_map.resize(clip_num); + for (int i = 0; i < clip_num; i++) { + intrinsics_map.resize(tracks.MaxFrame(i)+1); + for (int j = 0; j < intrinsics_map.size(); j++) { + intrinsics_map[i][j] = -1; + } + } +} + +void Reconstruction::InitIntrinsicsMapFixed(Tracks &tracks) { + int clip_num = tracks.MaxClip() + 1; + intrinsics_map.resize(clip_num); + for (int i = 0; i < clip_num; i++) { + intrinsics_map[i].resize(tracks.MaxFrame(i)+1); + for (int j = 0; j < intrinsics_map[i].size(); j++) { + intrinsics_map[i][j] = i; + } + } +} + +bool Reconstruction::SetIntrinsicsMap(int clip, int frame, int intrinsics) { + if (intrinsics_map.size() <= clip) + return false; + if (intrinsics_map[clip].size() <= frame) + return false; + intrinsics_map[clip][frame] = intrinsics; + return true; +} + +int Reconstruction::GetIntrinsicsMap(int clip, int frame) const { + if (intrinsics_map.size() <= clip) + return -1; + if (intrinsics_map[clip].size() <= frame) + return -1; + return intrinsics_map[clip][frame]; +} + +} // namespace mv diff --git a/intern/libmv/libmv/autotrack/reconstruction.h b/intern/libmv/libmv/autotrack/reconstruction.h index e1d4e882cbd..20eb3c2a1c4 100644 --- a/intern/libmv/libmv/autotrack/reconstruction.h +++ b/intern/libmv/libmv/autotrack/reconstruction.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014 libmv authors. +// Copyright (c) 2014, 2016 libmv authors. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to @@ -19,6 +19,7 @@ // IN THE SOFTWARE. // // Author: mierle@gmail.com (Keir Mierle) +// shentianweipku@gmail.com (Tianwei Shen) #ifndef LIBMV_AUTOTRACK_RECONSTRUCTION_H_ #define LIBMV_AUTOTRACK_RECONSTRUCTION_H_ @@ -31,10 +32,19 @@ namespace mv { using libmv::CameraIntrinsics; using libmv::vector; +using libmv::Mat3; +using libmv::Vec3; class Model; +struct Marker; +class Tracks; class CameraPose { +public: + CameraPose(): clip(-1), frame(-1) {} // uninitilized CameraPose is (-1, -1) + CameraPose(int clip_, int frame_, int intrinsics_, Mat3 R_, Vec3 t_): + clip(clip_), frame(frame_), intrinsics(intrinsics_), R(R_), t(t_) {} + int clip; int frame; int intrinsics; @@ -43,6 +53,10 @@ class CameraPose { }; class Point { +public: + Point(int track_ = -1, Vec3 X_ = Vec3(0, 0, 0)) + : track(track_), X(X_){} + Point(const Point &p) : track(p.track), X(p.X) {} int track; // The coordinates of the point. Note that not all coordinates are always @@ -51,39 +65,54 @@ class Point { }; // A reconstruction for a set of tracks. The indexing for clip, frame, and -// track should match that of a Tracs object, stored elsewhere. +// track should match that of a Tracks object, stored elsewhere. class Reconstruction { public: // All methods copy their input reference or take ownership of the pointer. void AddCameraPose(const CameraPose& pose); - int AddCameraIntrinsics(CameraIntrinsics* intrinsics); - int AddPoint(const Point& point); + int AddCameraIntrinsics(CameraIntrinsics* intrinsics_ptr); + int AddPoint(const Point& point); int AddModel(Model* model); // Returns the corresponding pose or point or NULL if missing. - CameraPose* CameraPoseForFrame(int clip, int frame); + CameraPose* CameraPoseForFrame(int clip, int frame); const CameraPose* CameraPoseForFrame(int clip, int frame) const; - Point* PointForTrack(int track); + Point* PointForTrack(int track); const Point* PointForTrack(int track) const; - const vector<vector<CameraPose> >& camera_poses() const { - return camera_poses_; - } + const vector<vector<CameraPose> >& camera_poses() const; + const vector<Point>& AllPoints() const; + + int GetClipNum() const; + int GetAllPoseNum() const; + int GetReconstructedCameraNum() const; - private: + // initialize all intrinsics map to -1 + void InitIntrinsicsMap(Tracks &tracks); + // initialize intrinsics of clip i to i(CameraPose::intrinsics) + void InitIntrinsicsMapFixed(Tracks &tracks); + // set CameraPose::intrinsics for frame (clip, frame) + bool SetIntrinsicsMap(int clip, int frame, int intrinsics); + // return CameraPose::intrinsics if (clip, frame) is intrinsics_map, otherwise return -1 + int GetIntrinsicsMap(int clip, int frame) const; + +private: // Indexed by CameraPose::intrinsics. Owns the intrinsics objects. vector<CameraIntrinsics*> camera_intrinsics_; - // Indexed by Marker::clip then by Marker::frame. vector<vector<CameraPose> > camera_poses_; - // Indexed by Marker::track. vector<Point> points_; - // Indexed by Marker::model_id. Owns model objects. vector<Model*> models_; + // Indexed by Marker::clip then by Marker::frame. + vector<vector<int> > intrinsics_map; }; +// Reconstruct two frames from the same clip, used as the initial reconstruction +bool ReconstructTwoFrames(const vector<Marker> &markers, + const int clip, + Reconstruction *reconstruction); } // namespace mv #endif // LIBMV_AUTOTRACK_RECONSTRUCTION_H_ diff --git a/intern/libmv/libmv/autotrack/resect.cc b/intern/libmv/libmv/autotrack/resect.cc new file mode 100644 index 00000000000..28392644553 --- /dev/null +++ b/intern/libmv/libmv/autotrack/resect.cc @@ -0,0 +1,187 @@ +// Copyright (c) 2011 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. + +#include "libmv/autotrack/resect.h" + +#include <cstdio> + +#include "libmv/base/vector.h" +#include "libmv/logging/logging.h" +#include "libmv/multiview/euclidean_resection.h" +#include "libmv/multiview/resection.h" +#include "libmv/multiview/projection.h" +#include "libmv/numeric/numeric.h" +#include "libmv/numeric/levenberg_marquardt.h" +#include "libmv/autotrack/reconstruction.h" +#include "libmv/autotrack/tracks.h" + +using libmv::Mat2X; +using libmv::Mat3X; +using libmv::Mat4X; +using libmv::Mat34; +using libmv::Vec; +using libmv::Vec6; + +namespace mv { +namespace { + +Mat2X PointMatrixFromMarkers(const vector<Marker> &markers) { + Mat2X points(2, markers.size()); + for (int i = 0; i < markers.size(); ++i) { + points(0, i) = markers[i].center[0]; + points(1, i) = markers[i].center[1]; + } + return points; +} + +// Uses an incremental rotation: +// +// x = R' * R * X + t; +// +// to avoid issues with the rotation representation. R' is derived from a +// euler vector encoding the rotation in 3 parameters; the direction is the +// axis to rotate around and the magnitude is the amount of the rotation. +struct EuclideanResectCostFunction { + public: + typedef Vec FMatrixType; + typedef Vec6 XMatrixType; + + EuclideanResectCostFunction(const vector<Marker> &markers, + const Reconstruction &reconstruction, + const Mat3 &initial_R) + : markers(markers), + reconstruction(reconstruction), + initial_R(initial_R) {} + + // dRt has dR (delta R) encoded as a euler vector in the first 3 parameters, + // followed by t in the next 3 parameters. + Vec operator()(const Vec6 &dRt) const { + // Unpack R, t from dRt. + Mat3 R = libmv::RotationFromEulerVector(dRt.head<3>()) * initial_R; + Vec3 t = dRt.tail<3>(); + + // Compute the reprojection error for each coordinate. + Vec residuals(2 * markers.size()); + residuals.setZero(); + for (int i = 0; i < markers.size(); ++i) { + const Point &point = + *reconstruction.PointForTrack(markers[i].track); + Vec3 projected = R * point.X + t; + projected /= projected(2); + residuals[2*i + 0] = projected(0) - markers[i].center[0]; + residuals[2*i + 1] = projected(1) - markers[i].center[1]; + } + return residuals; + } + + const vector<Marker> &markers; + const Reconstruction &reconstruction; + const Mat3 &initial_R; +}; + +} // namespace + +bool EuclideanResect(const vector<Marker> &markers, + Reconstruction *reconstruction, + bool final_pass, + int intrinsics) { + if (markers.size() < 5) { // five-point algorithm + return false; + } + Mat2X points_2d = PointMatrixFromMarkers(markers); + Mat3X points_3d(3, markers.size()); + for (int i = 0; i < markers.size(); i++) { + points_3d.col(i) = reconstruction->PointForTrack(markers[i].track)->X; + } + LG << "Number of points for resect: " << points_2d.cols() << "\n"; + + Mat3 R; + Vec3 t; + + if (0 || !libmv::euclidean_resection::EuclideanResection( + points_2d, points_3d, &R, &t, + libmv::euclidean_resection::RESECTION_EPNP)) { + LG << "[EuclideanResect] Euclidean resection failed\n"; + return false; + + if (!final_pass) return false; + // Euclidean resection failed. Fall back to projective resection, which is + // less reliable but better conditioned when there are many points. + Mat34 P; + Mat4X points_3d_homogeneous(4, markers.size()); + for (int i = 0; i < markers.size(); i++) { + points_3d_homogeneous.col(i).head<3>() = points_3d.col(i); + points_3d_homogeneous(3, i) = 1.0; + } + libmv::resection::Resection(points_2d, points_3d_homogeneous, &P); + if ((P * points_3d_homogeneous.col(0))(2) < 0) { + LG << "Point behind camera; switch sign."; + P = -P; + } + + Mat3 ignored; + libmv::KRt_From_P(P, &ignored, &R, &t); + + // The R matrix should be a rotation, but don't rely on it. + Eigen::JacobiSVD<Mat3> svd(R, Eigen::ComputeFullU | Eigen::ComputeFullV); + + LG << "Resection rotation is: " << svd.singularValues().transpose(); + LG << "Determinant is: " << R.determinant(); + + // Correct to make R a rotation. + R = svd.matrixU() * svd.matrixV().transpose(); + + Vec3 xx = R * points_3d.col(0) + t; + if (xx(2) < 0.0) { + LG << "Final point is still behind camera..."; + } + // XXX Need to check if error is horrible and fail here too in that case. + } + + // Refine the result. + typedef libmv::LevenbergMarquardt<EuclideanResectCostFunction> Solver; + + // Give the cost our initial guess for R. + EuclideanResectCostFunction resect_cost(markers, *reconstruction, R); + + // Encode the initial parameters: start with zero delta rotation, and the + // guess for t obtained from resection. + Vec6 dRt = Vec6::Zero(); + dRt.tail<3>() = t; + + Solver solver(resect_cost); + + Solver::SolverParameters params; + /* Solver::Results results = */ solver.minimize(params, &dRt); + VLOG(1) << "LM found incremental rotation: " << dRt.head<3>().transpose(); + // TODO(keir): Check results to ensure clean termination. + + // Unpack the rotation and translation. + R = libmv::RotationFromEulerVector(dRt.head<3>()) * R; + t = dRt.tail<3>(); + + VLOG(1) << "Resection for frame " << markers[0].clip << " " << markers[0].frame + << " got:\n" << "R:\n" << R << "\nt:\n" << t << "\n"; + CameraPose pose(markers[0].clip, markers[0].frame, intrinsics, R, t); + reconstruction->AddCameraPose(pose); + return true; +} + +} // namespace libmv diff --git a/intern/libmv/libmv/autotrack/resect.h b/intern/libmv/libmv/autotrack/resect.h new file mode 100644 index 00000000000..3d01f8d4f11 --- /dev/null +++ b/intern/libmv/libmv/autotrack/resect.h @@ -0,0 +1,61 @@ +// Copyright (c) 2016 libmv authors. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// Author: Tianwei Shen <shentianweipku@gmail.com> +// This file contains functions to do camera resection. + +#ifndef LIBMV_AUTOTRACK_RESECT_H +#define LIBMV_AUTOTRACK_RESECT_H + +#include "libmv/base/vector.h" +#include "libmv/autotrack/tracks.h" +#include "libmv/autotrack/reconstruction.h" + +namespace mv { + +/*! + Estimate the Euclidean pose of a camera from 2D to 3D correspondences. + + This takes a set of markers visible in one frame (which is the one to + resection), such that the markers are also reconstructed in 3D in the + reconstruction object, and solves for the pose and orientation of the + camera for that frame. + + \a markers should contain \l Marker markers \endlink belonging to tracks + visible in the one frame to be resectioned. Each of the tracks associated + with the markers must have a corresponding reconstructed 3D position in the + \a *reconstruction object. + + \a *reconstruction should contain the 3D points associated with the tracks + for the markers present in \a markers. + + \note This assumes a calibrated reconstruction, e.g. the markers are + already corrected for camera intrinsics and radial distortion. + \note This assumes an outlier-free set of markers. + + \return True if the resection was successful, false otherwise. + + \sa EuclideanIntersect, EuclideanReconstructTwoFrames +*/ +bool EuclideanResect(const vector<Marker> &markers, + Reconstruction *reconstruction, bool final_pass, int intrinsics); + +} // namespace mv + +#endif // LIBMV_AUTOTRACK_RESECT_H diff --git a/intern/libmv/libmv/autotrack/tracks.cc b/intern/libmv/libmv/autotrack/tracks.cc index 174f264f3f2..c72d0188769 100644 --- a/intern/libmv/libmv/autotrack/tracks.cc +++ b/intern/libmv/libmv/autotrack/tracks.cc @@ -78,7 +78,19 @@ void Tracks::GetMarkersInFrame(int clip, } } -void Tracks::GetMarkersForTracksInBothImages(int clip1, int frame1, +void Tracks::GetMarkersInBothFrames(int clip1, int frame1, + int clip2, int frame2, + vector<Marker> *markers) const { + for (int i = 0; i < markers_.size(); ++i) { + int clip = markers_[i].clip; + int frame = markers_[i].frame; + if ((clip == clip1 && frame == frame1) || + (clip == clip2 && frame == frame2)) + markers->push_back(markers_[i]); + } +} + +void Tracks::GetMarkersForTracksInBothFrames(int clip1, int frame1, int clip2, int frame2, vector<Marker>* markers) const { std::vector<int> image1_tracks; @@ -118,6 +130,12 @@ void Tracks::GetMarkersForTracksInBothImages(int clip1, int frame1, } } +void Tracks::GetAllMarkers(vector<Marker>* markers) const { + for (int i = 0; i < markers_.size(); ++i) { + markers->push_back(markers_[i]); + } +} + void Tracks::AddMarker(const Marker& marker) { // TODO(keir): This is quadratic for repeated insertions. Fix this by adding // a smarter data structure like a set<>. @@ -132,6 +150,13 @@ void Tracks::AddMarker(const Marker& marker) { markers_.push_back(marker); } +void Tracks::AddTracks(const Tracks& other_tracks) { + vector<Marker> markers; + other_tracks.GetAllMarkers(&markers); + for(int i = 0; i < markers.size(); ++i) + this->AddMarker(markers[i]); +} + void Tracks::SetMarkers(vector<Marker>* markers) { std::swap(markers_, *markers); } @@ -190,4 +215,19 @@ int Tracks::NumMarkers() const { return markers_.size(); } +void CoordinatesForMarkersInFrame(const vector<Marker> &markers, + int clip, int frame, + libmv::Mat *coordinates) { + vector<libmv::Vec2> coords; + for (int i = 0; i < markers.size(); ++i) { + const Marker &marker = markers[i]; + if (markers[i].clip == clip && markers[i].frame == frame) { + coords.push_back(libmv::Vec2(marker.center[0], marker.center[1])); + } + } + coordinates->resize(2, coords.size()); + for (int i = 0; i < coords.size(); i++) { + coordinates->col(i) = coords[i]; + } +} } // namespace mv diff --git a/intern/libmv/libmv/autotrack/tracks.h b/intern/libmv/libmv/autotrack/tracks.h index 0b7de91d211..8f63464a763 100644 --- a/intern/libmv/libmv/autotrack/tracks.h +++ b/intern/libmv/libmv/autotrack/tracks.h @@ -19,6 +19,7 @@ // IN THE SOFTWARE. // // Author: mierle@gmail.com (Keir Mierle) +// shentianweipku@gmail.com (Tianwei Shen) #ifndef LIBMV_AUTOTRACK_TRACKS_H_ #define LIBMV_AUTOTRACK_TRACKS_H_ @@ -36,30 +37,38 @@ class Tracks { Tracks() { } Tracks(const Tracks &other); - // Create a tracks object with markers already initialized. Copies markers. - explicit Tracks(const vector<Marker>& markers); + /// Create a tracks object with markers already initialized. Copies markers. + explicit Tracks(const vector<Marker> &markers); - // All getters append to the output argument vector. - bool GetMarker(int clip, int frame, int track, Marker* marker) const; - void GetMarkersForTrack(int track, vector<Marker>* markers) const; + /// All getters append to the output argument vector. + bool GetMarker(int clip, int frame, int track, Marker *marker) const; + void GetMarkersForTrack(int track, vector<Marker> *markers) const; void GetMarkersForTrackInClip(int clip, int track, vector<Marker>* markers) const; - void GetMarkersInFrame(int clip, int frame, vector<Marker>* markers) const; + void GetMarkersInFrame(int clip, int frame, vector<Marker> *markers) const; - // Get the markers in frame1 and frame2 which have a common track. - // + /// Returns all the markers visible in \a image1 and \a image2. + void GetMarkersInBothFrames(int clip1, int frame1, + int clip2, int frame2, + vector<Marker> *markers) const; + + /// Get the markers in frame1 and frame2 which have a common track. // This is not the same as the union of the markers in frame1 and // frame2; each marker is for a track that appears in both images. - void GetMarkersForTracksInBothImages(int clip1, int frame1, + void GetMarkersForTracksInBothFrames(int clip1, int frame1, int clip2, int frame2, - vector<Marker>* markers) const; + vector<Marker> *markers) const; + void GetAllMarkers(vector<Marker> *markers) const; + /// add a marker void AddMarker(const Marker& marker); + /// add markers from another Tracks + void AddTracks(const Tracks& other_tracks); // Moves the contents of *markers over top of the existing markers. This // destroys *markers in the process (but avoids copies). - void SetMarkers(vector<Marker>* markers); + void SetMarkers(vector<Marker> *markers); bool RemoveMarker(int clip, int frame, int track); void RemoveMarkersForTrack(int track); @@ -77,6 +86,9 @@ class Tracks { // linear lookup penalties for the accessors. }; +void CoordinatesForMarkersInFrame(const vector<Marker> &markers, + int clip, int frame, + libmv::Mat *coordinates); } // namespace mv #endif // LIBMV_AUTOTRACK_TRACKS_H_ diff --git a/release/scripts/startup/bl_ui/space_clip.py b/release/scripts/startup/bl_ui/space_clip.py index 3f05620fcf6..2a08d7af230 100644 --- a/release/scripts/startup/bl_ui/space_clip.py +++ b/release/scripts/startup/bl_ui/space_clip.py @@ -145,6 +145,39 @@ class CLIP_HT_header(Header): row.prop(toolsettings, "proportional_edit_falloff", text="", icon_only=True) + def _draw_correspondence(self, context): + layout = self.layout + + toolsettings = context.tool_settings + sc = context.space_data + clip = sc.clip + + row = layout.row(align=True) + row.template_header() + + CLIP_MT_correspondence_editor_menus.draw_collapsible(context, layout) + + row = layout.row() + row.template_ID(sc, "clip", open="clip.open") + + row = layout.row() # clip open for witness camera + row.template_ID(sc, "secondary_clip", open="clip.open_secondary") + + if clip: + tracking = clip.tracking + active_object = tracking.objects.active + + layout.prop(sc, "mode", text="") + layout.prop(sc, "view", text="", expand=True) + layout.prop(sc, "pivot_point", text="", icon_only=True) + + r = active_object.reconstruction + + if r.is_valid and sc.view == 'CLIP': + layout.label(text="Solve error: %.4f" % (r.average_error)) + else: + layout.prop(sc, "view", text="", expand=True) + def draw(self, context): layout = self.layout @@ -152,8 +185,10 @@ class CLIP_HT_header(Header): if sc.mode == 'TRACKING': self._draw_tracking(context) - else: + elif sc.mode == 'MASK': self._draw_masking(context) + else: # sc.mode == 'CORRESPONDENCE' + self._draw_correspondence(context) layout.template_running_jobs() @@ -205,6 +240,30 @@ class CLIP_MT_masking_editor_menus(Menu): layout.menu("CLIP_MT_clip") # XXX - remove? +class CLIP_MT_correspondence_editor_menus(Menu): + + bl_idname = "CLIP_MT_correspondence_editor_menus" + bl_label = "" + + def draw(self, context): + self.draw_menus(self.layout, context) + + @staticmethod + def draw_menus(layout, context): + sc = context.space_data + clip = sc.clip + + layout.menu("CLIP_MT_view") + + if clip: + layout.menu("CLIP_MT_select") + layout.menu("CLIP_MT_clip") + layout.menu("CLIP_MT_track") + layout.menu("CLIP_MT_reconstruction") + else: + layout.menu("CLIP_MT_clip") + + class CLIP_PT_clip_view_panel: @classmethod @@ -225,6 +284,16 @@ class CLIP_PT_tracking_panel: return clip and sc.mode == 'TRACKING' and sc.view == 'CLIP' +class CLIP_PT_correspondence_panel: + + @classmethod + def poll(cls, context): + sc = context.space_data + clip = sc.clip + + return clip and sc.mode == 'CORRESPONDENCE' and sc.view == 'CLIP' + + class CLIP_PT_reconstruction_panel: @classmethod @@ -400,6 +469,26 @@ class CLIP_PT_tools_tracking(CLIP_PT_tracking_panel, Panel): row.operator("clip.join_tracks", text="Join Tracks") +class CLIP_PT_tools_correspondence(CLIP_PT_correspondence_panel, Panel): + bl_space_type = 'CLIP_EDITOR' + bl_region_type = 'TOOLS' + bl_label = "Correspondence" + bl_category = "Track" + + def draw(self, context): + layout = self.layout + + col = layout.column(align=True) + row = col.row(align=True) + row.operator("clip.add_correspondence", text="Link") + row.operator("clip.delete_correspondence", text="Unlink") + + col = layout.column(align=True) + col.scale_y = 2.0 + + col.operator("clip.solve_multiview", text="Solve Multiview Camera") + + class CLIP_PT_tools_plane_tracking(CLIP_PT_tracking_panel, Panel): bl_space_type = 'CLIP_EDITOR' bl_region_type = 'TOOLS' diff --git a/source/blender/blenkernel/BKE_context.h b/source/blender/blenkernel/BKE_context.h index 4da6a61cbfa..0d758da0f53 100644 --- a/source/blender/blenkernel/BKE_context.h +++ b/source/blender/blenkernel/BKE_context.h @@ -50,6 +50,7 @@ struct ScrArea; struct SpaceLink; struct View3D; struct RegionView3D; +struct RegionSpaceClip; struct StructRNA; struct ToolSettings; struct Image; @@ -150,6 +151,7 @@ struct ReportList *CTX_wm_reports(const bContext *C); struct View3D *CTX_wm_view3d(const bContext *C); struct RegionView3D *CTX_wm_region_view3d(const bContext *C); +struct RegionSpaceClip *CTX_wm_region_clip(const bContext *C); struct SpaceText *CTX_wm_space_text(const bContext *C); struct SpaceImage *CTX_wm_space_image(const bContext *C); struct SpaceConsole *CTX_wm_space_console(const bContext *C); diff --git a/source/blender/blenkernel/BKE_tracking.h b/source/blender/blenkernel/BKE_tracking.h index 30873567297..814fb08d06c 100644 --- a/source/blender/blenkernel/BKE_tracking.h +++ b/source/blender/blenkernel/BKE_tracking.h @@ -34,9 +34,11 @@ struct bGPDlayer; struct ImBuf; -struct ListBase; +struct MovieClip; struct MovieReconstructContext; +struct MovieMultiviewReconstructContext; struct MovieTrackingTrack; +struct MovieTrackingCorrespondence; struct MovieTrackingMarker; struct MovieTrackingPlaneTrack; struct MovieTrackingPlaneMarker; @@ -288,6 +290,26 @@ void BKE_tracking_stabilization_data_to_mat4(int width, int height, float aspect void BKE_tracking_dopesheet_tag_update(struct MovieTracking *tracking); void BKE_tracking_dopesheet_update(struct MovieTracking *tracking); +/* Correspondence and multiview */ +void BKE_tracking_correspondence_unique_name(struct ListBase *tracksbase, struct MovieTrackingCorrespondence *corr); +struct MovieTrackingCorrespondence *BKE_tracking_correspondence_add(struct ListBase *corr_base, + struct MovieTrackingTrack *self_track, + struct MovieTrackingTrack *other_track, + struct MovieClip *self_clip, + struct MovieClip *other_clip, + char *error_msg, int error_size); +void BKE_tracking_multiview_reconstruction_solve(struct MovieMultiviewReconstructContext *context, short *stop, short *do_update, + float *progress, char *stats_message, int message_size); +struct MovieMultiviewReconstructContext *BKE_tracking_multiview_reconstruction_context_new(struct MovieClip **clips, + int num_clips, + struct MovieTrackingObject *object, + int keyframe1, int keyframe2, + int width, int height); +void BKE_tracking_multiview_reconstruction_context_free(struct MovieMultiviewReconstructContext *context); +bool BKE_tracking_multiview_reconstruction_check(struct MovieClip **clips, struct MovieTrackingObject *object, + char *error_msg, int error_size); +bool BKE_tracking_multiview_reconstruction_finish(struct MovieMultiviewReconstructContext *context, struct MovieClip** clips); + #define TRACK_SELECTED(track) ((track)->flag & SELECT || (track)->pat_flag & SELECT || (track)->search_flag & SELECT) #define TRACK_AREA_SELECTED(track, area) ((area) == TRACK_AREA_POINT ? (track)->flag & SELECT : \ diff --git a/source/blender/blenkernel/CMakeLists.txt b/source/blender/blenkernel/CMakeLists.txt index 157c4408d6a..9af1afed338 100644 --- a/source/blender/blenkernel/CMakeLists.txt +++ b/source/blender/blenkernel/CMakeLists.txt @@ -177,6 +177,7 @@ set(SRC intern/texture.c intern/tracking.c intern/tracking_auto.c + intern/tracking_correspondence.c intern/tracking_detect.c intern/tracking_plane_tracker.c intern/tracking_region_tracker.c diff --git a/source/blender/blenkernel/intern/context.c b/source/blender/blenkernel/intern/context.c index 4c01bfd35f2..bd117e64e99 100644 --- a/source/blender/blenkernel/intern/context.c +++ b/source/blender/blenkernel/intern/context.c @@ -686,6 +686,37 @@ RegionView3D *CTX_wm_region_view3d(const bContext *C) return NULL; } +RegionSpaceClip *CTX_wm_region_clip(const bContext *C) +{ + ScrArea *sa = CTX_wm_area(C); + ARegion *ar_curr = CTX_wm_region(C); + + if (sa && sa->spacetype == SPACE_CLIP) { + /* only RGN_TYPE_WINDOW has regiondata */ + if (ar_curr && ar_curr->regiontype == RGN_TYPE_WINDOW) { + return ar_curr->regiondata; + } + else { + /* search forward and backward to find regiondata */ + ARegion *ar = ar_curr->prev; + while (ar) { + if (ar->regiontype == RGN_TYPE_WINDOW) { + return ar->regiondata; + } + ar = ar->prev; + } + ar = ar_curr->next; + while (ar) { + if (ar->regiontype == RGN_TYPE_WINDOW) { + return ar->regiondata; + } + ar = ar->next; + } + } + } + return NULL; +} + struct SpaceText *CTX_wm_space_text(const bContext *C) { ScrArea *sa = CTX_wm_area(C); diff --git a/source/blender/blenkernel/intern/tracking.c b/source/blender/blenkernel/intern/tracking.c index 990d250b854..7b11d428757 100644 --- a/source/blender/blenkernel/intern/tracking.c +++ b/source/blender/blenkernel/intern/tracking.c @@ -619,6 +619,16 @@ void BKE_tracking_track_unique_name(ListBase *tracksbase, MovieTrackingTrack *tr offsetof(MovieTrackingTrack, name), sizeof(track->name)); } +/* Ensure specified correspondence has got unique name, + * if it's not name of specified correspondence will be changed + * keeping names of all other correspondence unchanged. + */ +void BKE_tracking_correspondence_unique_name(ListBase *tracksbase, MovieTrackingCorrespondence *corr) +{ + BLI_uniquename(tracksbase, corr, CTX_DATA_(BLT_I18NCONTEXT_ID_MOVIECLIP, "Correspondence"), '.', + offsetof(MovieTrackingCorrespondence, name), sizeof(corr->name)); +} + /* Free specified track, only frees contents of a structure * (if track is allocated in heap, it shall be handled outside). * diff --git a/source/blender/blenkernel/intern/tracking_correspondence.c b/source/blender/blenkernel/intern/tracking_correspondence.c new file mode 100644 index 00000000000..2a504a5fee7 --- /dev/null +++ b/source/blender/blenkernel/intern/tracking_correspondence.c @@ -0,0 +1,705 @@ +/* + * ***** BEGIN GPL LICENSE BLOCK ***** + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2011 Blender Foundation. + * All rights reserved. + * + * Contributor(s): Blender Foundation, + * Tianwei Shen + * + * ***** END GPL LICENSE BLOCK ***** + */ + +/** \file blender/blenkernel/intern/tracking_correspondence.c + * \ingroup bke + * + * This file contains blender-side correspondence functions for witness camera support + */ + +#include <limits.h> + +#include "MEM_guardedalloc.h" +#include "DNA_movieclip_types.h" +#include "DNA_object_types.h" /* SELECT */ +#include "DNA_anim_types.h" + +#include "BLI_utildefines.h" +#include "BLI_listbase.h" +#include "BLI_string.h" +#include "BLI_math.h" +#include "BLI_ghash.h" +#include "BLT_translation.h" + +#include "BKE_tracking.h" +#include "BKE_fcurve.h" +#include "BKE_movieclip.h" + +#include "RNA_access.h" + +#include "IMB_imbuf_types.h" + +#include "libmv-capi.h" +#include "tracking_private.h" + +struct ReconstructProgressData; + +typedef struct MovieMultiviewReconstructContext { + int clip_num; /* number of clips in this reconstruction */ + struct libmv_TracksN **all_tracks; /* set of tracks from all clips (API in autotrack) */ + // TODO(tianwei): might be proper to make it libmv_multiview_Reconstruction + struct libmv_ReconstructionN **all_reconstruction; /* reconstruction for each clip (API in autotrack) */ + libmv_CameraIntrinsicsOptions *all_camera_intrinsics_options; /* camera intrinsic of each camera */ + TracksMap **all_tracks_map; /* tracks_map of each clip */ + int *all_sfra, *all_efra; /* start and end frame of each clip */ + int *all_refine_flags; /* refine flags of each clip */ + int **track_global_index; /* track global index */ + + bool select_keyframes; + int keyframe1, keyframe2; /* the key frames selected from the primary camera */ + char object_name[MAX_NAME]; + bool is_camera; + short motion_flag; + float reprojection_error; /* average reprojection error for all clips and tracks */ +} MovieMultiviewReconstructContext; + +typedef struct MultiviewReconstructProgressData { + short *stop; + short *do_update; + float *progress; + char *stats_message; + int message_size; +} MultiviewReconstructProgressData; + +/* Add new correspondence to a specified correspondence base. + */ +MovieTrackingCorrespondence *BKE_tracking_correspondence_add(ListBase *corr_base, + MovieTrackingTrack *self_track, + MovieTrackingTrack *other_track, + MovieClip* self_clip, + MovieClip* other_clip, + char *error_msg, int error_size) +{ + MovieTrackingCorrespondence *corr = NULL; + /* check self correspondences */ + if (self_track == other_track) { + BLI_strncpy(error_msg, N_("Cannot link a track to itself"), error_size); + return NULL; + } + /* check duplicate correspondences or conflict correspondence */ + for (corr = corr_base->first; corr != NULL; corr = corr->next) + { + if (corr->self_clip == self_clip && strcmp(corr->self_track_name, self_track->name) == 0) { + /* duplicate correspondences */ + if (corr->other_clip == other_clip && strcmp(corr->other_track_name, other_track->name) == 0) { + BLI_strncpy(error_msg, N_("This correspondence has been added"), error_size); + return NULL; + } + /* conflict correspondence */ + else { + BLI_strncpy(error_msg, N_("Conflict correspondence, consider first deleting the old one"), error_size); + return NULL; + } + } + if (corr->other_clip == other_clip && strcmp(corr->other_track_name, other_track->name) == 0) { + if (corr->self_clip == self_clip && strcmp(corr->self_track_name, self_track->name) == 0) { + BLI_strncpy(error_msg, N_("This correspondence has been added"), error_size); + return NULL; + } + else { + BLI_strncpy(error_msg, N_("Conflict correspondence, consider first deleting the old one"), error_size); + return NULL; + } + } + } + + corr = MEM_callocN(sizeof(MovieTrackingCorrespondence), "add correspondence"); + strcpy(corr->name, "Correspondence"); + strcpy(corr->self_track_name, self_track->name); + strcpy(corr->other_track_name, other_track->name); + corr->self_clip = self_clip; + corr->other_clip = other_clip; + + BLI_addtail(corr_base, corr); + BKE_tracking_correspondence_unique_name(corr_base, corr); + + return corr; +} + +/* Convert blender's multiview refinement flags to libmv's. + * These refined flags would be the same as the single view version + */ +static int multiview_refine_intrinsics_get_flags(MovieTracking *tracking, MovieTrackingObject *object) +{ + int refine = tracking->settings.refine_camera_intrinsics; + int flags = 0; + + if ((object->flag & TRACKING_OBJECT_CAMERA) == 0) + return 0; + + if (refine & REFINE_FOCAL_LENGTH) + flags |= LIBMV_REFINE_FOCAL_LENGTH; + + if (refine & REFINE_PRINCIPAL_POINT) + flags |= LIBMV_REFINE_PRINCIPAL_POINT; + + if (refine & REFINE_RADIAL_DISTORTION_K1) + flags |= LIBMV_REFINE_RADIAL_DISTORTION_K1; + + if (refine & REFINE_RADIAL_DISTORTION_K2) + flags |= LIBMV_REFINE_RADIAL_DISTORTION_K2; + + return flags; +} + +/* Create new libmv Tracks structure from blender's tracks list. */ +static struct libmv_TracksN *libmv_multiview_tracks_new(MovieClip *clip, int clip_id, ListBase *tracksbase, + int *global_track_index, int width, int height) +{ + int tracknr = 0; + MovieTrackingTrack *track; + struct libmv_TracksN *tracks = libmv_tracksNewN(); + + track = tracksbase->first; + while (track) { + FCurve *weight_fcurve; + int a = 0; + + weight_fcurve = id_data_find_fcurve(&clip->id, track, &RNA_MovieTrackingTrack, + "weight", 0, NULL); + + for (a = 0; a < track->markersnr; a++) { + MovieTrackingMarker *marker = &track->markers[a]; + + if ((marker->flag & MARKER_DISABLED) == 0) { + float weight = track->weight; + + if (weight_fcurve) { + int scene_framenr = + BKE_movieclip_remap_clip_to_scene_frame(clip, marker->framenr); + weight = evaluate_fcurve(weight_fcurve, scene_framenr); + } + + libmv_Marker libmv_marker; + libmv_marker.clip = clip_id; + libmv_marker.frame = marker->framenr; + libmv_marker.track = global_track_index[tracknr]; + libmv_marker.center[0] = (marker->pos[0] + track->offset[0]) * width; + libmv_marker.center[1] = (marker->pos[1] + track->offset[1]) * height; + for (int i = 0; i < 4; i++) { + libmv_marker.patch[i][0] = marker->pattern_corners[i][0]; + libmv_marker.patch[i][1] = marker->pattern_corners[i][1]; + } + for (int i = 0; i < 2; i++) { + libmv_marker.search_region_min[i] = marker->search_min[i]; + libmv_marker.search_region_max[i] = marker->search_max[i]; + } + libmv_marker.weight = weight; + + // the following the unused in the current pipeline + // TODO(tianwei): figure out how to fill in reference clip and frame + if (marker->flag & MARKER_TRACKED) { + libmv_marker.source = LIBMV_MARKER_SOURCE_TRACKED; + } + else { + libmv_marker.source = LIBMV_MARKER_SOURCE_MANUAL; + } + libmv_marker.status = LIBMV_MARKER_STATUS_UNKNOWN; + libmv_marker.reference_clip = clip_id; + libmv_marker.reference_frame = -1; + + libmv_marker.model_type = LIBMV_MARKER_MODEL_TYPE_POINT; + libmv_marker.model_id = 0; + libmv_marker.disabled_channels = + ((track->flag & TRACK_DISABLE_RED) ? LIBMV_MARKER_CHANNEL_R : 0) | + ((track->flag & TRACK_DISABLE_GREEN) ? LIBMV_MARKER_CHANNEL_G : 0) | + ((track->flag & TRACK_DISABLE_BLUE) ? LIBMV_MARKER_CHANNEL_B : 0); + + //TODO(tianwei): why some framenr is negative???? + if (clip_id < 0 || marker->framenr < 0) + continue; + libmv_tracksAddMarkerN(tracks, &libmv_marker); + } + } + + track = track->next; + tracknr++; + } + + return tracks; +} + +/* get correspondences from blender tracking to libmv correspondences + * return the number of correspondences converted + */ +static int libmv_CorrespondencesFromTracking(ListBase *tracking_correspondences, + MovieClip **clips, + const int clip_num, + int **global_track_index) +{ + int num_valid_corrs = 0; + MovieTrackingCorrespondence *corr; + corr = tracking_correspondences->first; + while (corr) { + int clip1 = -1, clip2 = -1, track1 = -1, track2 = -1; + MovieClip *self_clip = corr->self_clip; + MovieClip *other_clip = corr->other_clip; + /* iterate through all the clips to get the local clip id */ + for (int i = 0; i < clip_num; i++) { + MovieTracking *tracking = &clips[i]->tracking; + ListBase *tracksbase = &tracking->tracks; + MovieTrackingTrack *track = tracksbase->first; + int tracknr = 0; + /* check primary clip */ + if (self_clip == clips[i]) { + clip1 = i; + while (track) { + if (strcmp(corr->self_track_name, track->name) == 0) { + track1 = tracknr; + break; + } + track = track->next; + tracknr++; + } + } + /* check witness clip */ + if (other_clip == clips[i]) { + clip2 = i; + while (track) { + if (strcmp(corr->other_track_name, track->name) == 0) { + track2 = tracknr; + break; + } + track = track->next; + tracknr++; + } + } + } + if (clip1 != -1 && clip2 != -1 && track1 != -1 && track2 != -1 && clip1 != clip2) { + num_valid_corrs++; + } + /* change the global index of clip2-track2 to clip1-track1 */ + global_track_index[clip2][track2] = global_track_index[clip1][track1]; + corr = corr->next; + } + return num_valid_corrs; +} + +/* Create context for camera/object motion reconstruction. + * Copies all data needed for reconstruction from movie + * clip datablock, so editing this clip is safe during + * reconstruction job is in progress. + */ +MovieMultiviewReconstructContext * +BKE_tracking_multiview_reconstruction_context_new(MovieClip **clips, + int num_clips, + MovieTrackingObject *object, + int keyframe1, int keyframe2, + int width, int height) +{ + MovieMultiviewReconstructContext *context = MEM_callocN(sizeof(MovieMultiviewReconstructContext), + "MRC data"); + // alloc memory for the field members + context->all_tracks = MEM_callocN(num_clips * sizeof(libmv_TracksN*), "MRC libmv_Tracks"); + context->all_reconstruction = MEM_callocN(num_clips * sizeof(struct libmv_ReconstructionN*), "MRC libmv reconstructions"); + context->all_tracks_map = MEM_callocN(num_clips * sizeof(TracksMap*), "MRC TracksMap"); + context->all_camera_intrinsics_options = MEM_callocN(num_clips * sizeof(libmv_CameraIntrinsicsOptions), "MRC camera intrinsics"); + context->all_sfra = MEM_callocN(num_clips * sizeof(int), "MRC start frames"); + context->all_efra = MEM_callocN(num_clips * sizeof(int), "MRC end frames"); + context->all_refine_flags = MEM_callocN(num_clips * sizeof(int), "MRC refine flags"); + context->keyframe1 = keyframe1; + context->keyframe2 = keyframe2; + context->clip_num = num_clips; + + // initial global track index to [0,..., N1 - 1], [N1,..., N1+N2-1], so on and so forth + context->track_global_index = MEM_callocN(num_clips * sizeof(int*), "global track index of each clip"); + int global_index = 0; + for (int i = 0; i < num_clips; i++) { + MovieClip *clip = clips[i]; + MovieTracking *tracking = &clip->tracking; + ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object); + int num_tracks = BLI_listbase_count(tracksbase); + context->track_global_index[i] = MEM_callocN(num_tracks * sizeof(int), "global track index for clip i"); + for (int j = 0; j < num_tracks; j++) { + context->track_global_index[i][j] = global_index++; + } + } + + for (int i = 0; i < num_clips; i++) { + MovieClip *clip = clips[i]; + MovieTracking *tracking = &clip->tracking; + ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object); + float aspy = 1.0f / tracking->camera.pixel_aspect; + int num_tracks = BLI_listbase_count(tracksbase); + if (i == 0) + printf("%d tracks in the primary clip 0\n", num_tracks); + else + printf("%d tracks in the witness camera %d\n", num_tracks, i); + int sfra = INT_MAX, efra = INT_MIN; + MovieTrackingTrack *track; + + // setting context only from information in the primary clip + if (i == 0) { + // correspondences are recorded in the primary clip (0), convert local track id to global track id + int num_valid_corrs = libmv_CorrespondencesFromTracking(&tracking->correspondences, clips, num_clips, + context->track_global_index); + printf("num valid corrs: %d\n", num_valid_corrs); + BLI_assert(num_valid_corrs == BLI_listbase_count(&tracking->correspondences)); + + BLI_strncpy(context->object_name, object->name, sizeof(context->object_name)); + context->is_camera = object->flag & TRACKING_OBJECT_CAMERA; + context->motion_flag = tracking->settings.motion_flag; + context->select_keyframes = + (tracking->settings.reconstruction_flag & TRACKING_USE_KEYFRAME_SELECTION) != 0; + } + + tracking_cameraIntrinscisOptionsFromTracking(tracking, + width, height, + &context->all_camera_intrinsics_options[i]); + + context->all_tracks_map[i] = tracks_map_new(context->object_name, context->is_camera, num_tracks, 0); + context->all_refine_flags[i] = multiview_refine_intrinsics_get_flags(tracking, object); + + track = tracksbase->first; + while (track) { + int first = 0, last = track->markersnr - 1; + MovieTrackingMarker *first_marker = &track->markers[0]; + MovieTrackingMarker *last_marker = &track->markers[track->markersnr - 1]; + + /* find first not-disabled marker */ + while (first <= track->markersnr - 1 && first_marker->flag & MARKER_DISABLED) { + first++; + first_marker++; + } + + /* find last not-disabled marker */ + while (last >= 0 && last_marker->flag & MARKER_DISABLED) { + last--; + last_marker--; + } + + if (first <= track->markersnr - 1) + sfra = min_ii(sfra, first_marker->framenr); + + if (last >= 0) + efra = max_ii(efra, last_marker->framenr); + + tracks_map_insert(context->all_tracks_map[i], track, NULL); + + track = track->next; + } + context->all_sfra[i] = sfra; + context->all_efra[i] = efra; + context->all_tracks[i] = libmv_multiview_tracks_new(clip, i, tracksbase, context->track_global_index[i], + width, height * aspy); + } + + return context; +} + +/* Free memory used by a reconstruction process. */ +void BKE_tracking_multiview_reconstruction_context_free(MovieMultiviewReconstructContext *context) +{ + for (int i = 0; i < context->clip_num; i++) { + libmv_tracksDestroyN(context->all_tracks[i]); + if (context->all_reconstruction[i]) + libmv_reconstructionNDestroy(context->all_reconstruction[i]); + if (context->track_global_index[i]) + MEM_freeN(context->track_global_index[i]); + tracks_map_free(context->all_tracks_map[i], NULL); + } + MEM_freeN(context->all_tracks); + MEM_freeN(context->all_reconstruction); + MEM_freeN(context->all_camera_intrinsics_options); + MEM_freeN(context->all_tracks_map); + MEM_freeN(context->all_sfra); + MEM_freeN(context->all_efra); + MEM_freeN(context->all_refine_flags); + MEM_freeN(context->track_global_index); + + MEM_freeN(context); +} + + +/* Fill in multiview reconstruction options structure from reconstruction context. */ +static void multiviewReconstructionOptionsFromContext(libmv_MultiviewReconstructionOptions *reconstruction_options, + MovieMultiviewReconstructContext *context) +{ + reconstruction_options->select_keyframes = context->select_keyframes; + + reconstruction_options->keyframe1 = context->keyframe1; + reconstruction_options->keyframe2 = context->keyframe2; + + reconstruction_options->all_refine_intrinsics = context->all_refine_flags; +} + +/* Callback which is called from libmv side to update progress in the interface. */ +static void multiview_reconstruct_update_solve_cb(void *customdata, double progress, const char *message) +{ + MultiviewReconstructProgressData *progressdata = customdata; + + if (progressdata->progress) { + *progressdata->progress = progress; + *progressdata->do_update = true; + } + + BLI_snprintf(progressdata->stats_message, progressdata->message_size, "Solving cameras | %s", message); +} + +/* stop is not actually used at this moment, so reconstruction + * job could not be stopped. + * + * do_update, progress and stat_message are set by reconstruction + * callback in libmv side and passing to an interface. + */ +void BKE_tracking_multiview_reconstruction_solve(MovieMultiviewReconstructContext *context, short *stop, short *do_update, + float *progress, char *stats_message, int message_size) +{ + float error; + + MultiviewReconstructProgressData progressdata; + + libmv_MultiviewReconstructionOptions reconstruction_options; + + progressdata.stop = stop; + progressdata.do_update = do_update; + progressdata.progress = progress; + progressdata.stats_message = stats_message; + progressdata.message_size = message_size; + + multiviewReconstructionOptionsFromContext(&reconstruction_options, context); + + if (context->motion_flag & TRACKING_MOTION_MODAL) { + // TODO(tianwei): leave tracking solve object for now + //context->reconstruction = libmv_solveModal(context->tracks, + // &context->camera_intrinsics_options, + // &reconstruction_options, + // multiview_reconstruct_update_solve_cb, &progressdata); + } + else { + context->all_reconstruction = libmv_solveMultiviewReconstruction( + context->clip_num, + (const libmv_TracksN **) context->all_tracks, + (const libmv_CameraIntrinsicsOptions *) context->all_camera_intrinsics_options, + &reconstruction_options, + multiview_reconstruct_update_solve_cb, &progressdata); + + if (context->select_keyframes) { + /* store actual keyframes used for reconstruction to update them in the interface later */ + context->keyframe1 = reconstruction_options.keyframe1; + context->keyframe2 = reconstruction_options.keyframe2; + } + } + + error = libmv_multiviewReprojectionError(context->clip_num, + (const libmv_ReconstructionN**)context->all_reconstruction); + + context->reprojection_error = error; +} + +/* Retrieve multiview refined camera intrinsics from libmv to blender. */ +static void multiview_reconstruct_retrieve_libmv_intrinsics(MovieMultiviewReconstructContext *context, + int clip_id, + MovieTracking *tracking) +{ + struct libmv_ReconstructionN *libmv_reconstruction = context->all_reconstruction[clip_id]; + struct libmv_CameraIntrinsics *libmv_intrinsics = libmv_reconstructionNExtractIntrinsics(libmv_reconstruction); + + libmv_CameraIntrinsicsOptions camera_intrinsics_options; + libmv_cameraIntrinsicsExtractOptions(libmv_intrinsics, &camera_intrinsics_options); + + tracking_trackingCameraFromIntrinscisOptions(tracking, + &camera_intrinsics_options); +} + +/* Retrieve multiview reconstructed tracks from libmv to blender. + * and also copies reconstructed cameras from libmv to movie clip datablock. + */ +static bool multiview_reconstruct_retrieve_libmv_info(MovieMultiviewReconstructContext *context, + int clip_id, + MovieTracking *tracking) +{ + // libmv reconstruction results in saved in context->all_reconstruction[0] + struct libmv_ReconstructionN *libmv_reconstruction = context->all_reconstruction[0]; + MovieTrackingReconstruction *reconstruction = NULL; + MovieReconstructedCamera *reconstructed; + MovieTrackingTrack *track; + ListBase *tracksbase = NULL; + int tracknr = 0, a; + bool ok = true; + bool origin_set = false; + int sfra = context->all_sfra[clip_id], efra = context->all_efra[clip_id]; + float imat[4][4]; + + if (context->is_camera) { + tracksbase = &tracking->tracks; + reconstruction = &tracking->reconstruction; + } + else { + MovieTrackingObject *object = BKE_tracking_object_get_named(tracking, context->object_name); + tracksbase = &object->tracks; + reconstruction = &object->reconstruction; + } + + unit_m4(imat); + + int *track_index_map = context->track_global_index[clip_id]; // this saves the global track index mapping + track = tracksbase->first; + while (track) { + double pos[3]; + + if (libmv_multiviewPointForTrack(libmv_reconstruction, track_index_map[tracknr], pos)) { + track->bundle_pos[0] = pos[0]; + track->bundle_pos[1] = pos[1]; + track->bundle_pos[2] = pos[2]; + + track->flag |= TRACK_HAS_BUNDLE; + track->error = libmv_multiviewReprojectionErrorForTrack(libmv_reconstruction, track_index_map[tracknr]); + } + else { + track->flag &= ~TRACK_HAS_BUNDLE; + ok = false; + + printf("Unable to reconstruct position for track #%d '%s'\n", tracknr, track->name); + } + + track = track->next; + tracknr++; + } + + if (reconstruction->cameras) + MEM_freeN(reconstruction->cameras); + + reconstruction->camnr = 0; + reconstruction->cameras = NULL; + reconstructed = MEM_callocN((efra - sfra + 1) * sizeof(MovieReconstructedCamera), + "temp reconstructed camera"); + + for (a = sfra; a <= efra; a++) { + double matd[4][4]; + + if (libmv_multiviewCameraForFrame(libmv_reconstruction, clip_id, a, matd)) { + int i, j; + float mat[4][4]; + float error = libmv_multiviewReprojectionErrorForFrame(libmv_reconstruction, clip_id, a); + + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) + mat[i][j] = matd[i][j]; + } + + /* Ensure first camera has got zero rotation and transform. + * This is essential for object tracking to work -- this way + * we'll always know object and environment are properly + * oriented. + * + * There's one weak part tho, which is requirement object + * motion starts at the same frame as camera motion does, + * otherwise that;' be a russian roulette whether object is + * aligned correct or not. + */ + if (!origin_set) { + invert_m4_m4(imat, mat); + unit_m4(mat); + origin_set = true; + } + else { + mul_m4_m4m4(mat, imat, mat); + } + + copy_m4_m4(reconstructed[reconstruction->camnr].mat, mat); + reconstructed[reconstruction->camnr].framenr = a; + reconstructed[reconstruction->camnr].error = error; + reconstruction->camnr++; + } + else { + ok = false; + printf("No camera for frame %d %d\n", clip_id, a); + } + } + + if (reconstruction->camnr) { + int size = reconstruction->camnr * sizeof(MovieReconstructedCamera); + reconstruction->cameras = MEM_callocN(size, "reconstructed camera"); + memcpy(reconstruction->cameras, reconstructed, size); + } + + if (origin_set) { + track = tracksbase->first; + while (track) { + if (track->flag & TRACK_HAS_BUNDLE) + mul_v3_m4v3(track->bundle_pos, imat, track->bundle_pos); + + track = track->next; + } + } + + MEM_freeN(reconstructed); + + return ok; +} + +/* Retrieve all the multiview reconstruction libmv data from context to blender's side data blocks. */ +static int multiview_reconstruct_retrieve_libmv(MovieMultiviewReconstructContext *context, + int clip_id, + MovieTracking *tracking) +{ + /* take the intrinsics back from libmv */ + multiview_reconstruct_retrieve_libmv_intrinsics(context, clip_id, tracking); + + /* take reconstructed camera and tracks info back from libmv */ + return multiview_reconstruct_retrieve_libmv_info(context, clip_id, tracking); +} + +/* Finish multiview reconstruction process by copying reconstructed data + * to the each movie clip datablock. + */ +bool BKE_tracking_multiview_reconstruction_finish(MovieMultiviewReconstructContext *context, MovieClip** clips) +{ + if (!libmv_multiviewReconstructionIsValid(context->clip_num, + (const libmv_ReconstructionN**) context->all_reconstruction)) { + printf("Failed solve the multiview motion: at least one clip failed\n"); + return false; + } + + for (int i = 0; i < context->clip_num; i++) { + MovieTrackingReconstruction *reconstruction; + MovieTrackingObject *object; + + MovieClip *clip = clips[i]; + MovieTracking *tracking = &clip->tracking; + tracks_map_merge(context->all_tracks_map[i], tracking); + BKE_tracking_dopesheet_tag_update(tracking); + + object = BKE_tracking_object_get_named(tracking, context->object_name); + if (context->is_camera) + reconstruction = &tracking->reconstruction; + else + reconstruction = &object->reconstruction; + ///* update keyframe in the interface */ + if (context->select_keyframes) { + object->keyframe1 = context->keyframe1; + object->keyframe2 = context->keyframe2; + } + reconstruction->error = context->reprojection_error; + reconstruction->flag |= TRACKING_RECONSTRUCTED; + + if (!multiview_reconstruct_retrieve_libmv(context, i, tracking)) + return false; + } + + return true; +} diff --git a/source/blender/blenkernel/intern/tracking_solver.c b/source/blender/blenkernel/intern/tracking_solver.c index c21883c6eb8..3a2ce5b80d3 100644 --- a/source/blender/blenkernel/intern/tracking_solver.c +++ b/source/blender/blenkernel/intern/tracking_solver.c @@ -581,3 +581,33 @@ void BKE_tracking_reconstruction_scale(MovieTracking *tracking, float scale[3]) tracking_scale_reconstruction(tracksbase, reconstruction, scale); } } + +/********************** Multi-view reconstruction functions *********************/ + +/* Perform early check on whether everything is fine to start reconstruction. */ +bool BKE_tracking_multiview_reconstruction_check(MovieClip **clips, MovieTrackingObject *object, + char *error_msg, int error_size) +{ +#ifndef WITH_LIBMV + BLI_strncpy(error_msg, N_("Blender is compiled without motion tracking library"), error_size); + return false; +#endif + MovieClip *primary_clip = clips[0]; + MovieTracking *tracking = &primary_clip->tracking; + if (tracking->settings.motion_flag & TRACKING_MOTION_MODAL) { + /* TODO: check for number of tracks? */ + return true; + } + else if ((tracking->settings.reconstruction_flag & TRACKING_USE_KEYFRAME_SELECTION) == 0) { + /* automatic keyframe selection does not require any pre-process checks */ + if (reconstruct_count_tracks_on_both_keyframes(tracking, object) < 8) { + BLI_strncpy(error_msg, + N_("At least 8 common tracks on both keyframes are needed for reconstruction"), + error_size); + + return false; + } + } + + return true; +} diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c index c55b426c025..2900f9e4fc4 100644 --- a/source/blender/blenloader/intern/readfile.c +++ b/source/blender/blenloader/intern/readfile.c @@ -6560,6 +6560,7 @@ static void lib_link_screen(FileData *fd, Main *main) SpaceClip *sclip = (SpaceClip *)sl; sclip->clip = newlibadr_real_us(fd, sc->id.lib, sclip->clip); + sclip->secondary_clip = newlibadr_real_us(fd, sc->id.lib, sclip->secondary_clip); sclip->mask_info.mask = newlibadr_real_us(fd, sc->id.lib, sclip->mask_info.mask); break; } @@ -6949,6 +6950,7 @@ void blo_lib_link_screen_restore(Main *newmain, bScreen *curscreen, Scene *cursc SpaceClip *sclip = (SpaceClip *)sl; sclip->clip = restore_pointer_by_name(id_map, (ID *)sclip->clip, USER_REAL); + sclip->secondary_clip = restore_pointer_by_name(id_map, (ID *)sclip->secondary_clip, USER_REAL); sclip->mask_info.mask = restore_pointer_by_name(id_map, (ID *)sclip->mask_info.mask, USER_REAL); sclip->scopes.ok = 0; @@ -7588,6 +7590,12 @@ static void direct_link_moviePlaneTracks(FileData *fd, ListBase *plane_tracks_ba } } +static void direct_link_movieCorrespondences(FileData *fd, + ListBase *correspondences) +{ + link_list(fd, correspondences); +} + static void direct_link_movieclip(FileData *fd, MovieClip *clip) { MovieTracking *tracking = &clip->tracking; @@ -7603,6 +7611,7 @@ static void direct_link_movieclip(FileData *fd, MovieClip *clip) direct_link_movieTracks(fd, &tracking->tracks); direct_link_moviePlaneTracks(fd, &tracking->plane_tracks); + direct_link_movieCorrespondences(fd, &tracking->correspondences); direct_link_movieReconstruction(fd, &tracking->reconstruction); clip->tracking.act_track = newdataadr(fd, clip->tracking.act_track); @@ -7646,6 +7655,17 @@ static void lib_link_moviePlaneTracks(FileData *fd, MovieClip *clip, ListBase *t } } +static void lib_link_movieCorrespondences(FileData *fd, + MovieClip *clip, + ListBase *correspondences) +{ + MovieTrackingCorrespondence *corr; + for (corr = correspondences->first; corr != NULL; corr = corr->next) { + corr->other_clip = newlibadr(fd, clip->id.lib, corr->other_clip); + corr->self_clip = newlibadr(fd, clip->id.lib, corr->self_clip); + } +} + static void lib_link_movieclip(FileData *fd, Main *main) { for (MovieClip *clip = main->movieclip.first; clip; clip = clip->id.next) { @@ -7659,6 +7679,7 @@ static void lib_link_movieclip(FileData *fd, Main *main) lib_link_movieTracks(fd, clip, &tracking->tracks); lib_link_moviePlaneTracks(fd, clip, &tracking->plane_tracks); + lib_link_movieCorrespondences(fd, clip, &tracking->correspondences); for (MovieTrackingObject *object = tracking->objects.first; object; object = object->next) { lib_link_movieTracks(fd, clip, &object->tracks); diff --git a/source/blender/blenloader/intern/versioning_270.c b/source/blender/blenloader/intern/versioning_270.c index d3f33cf725f..8eb61251ddd 100644 --- a/source/blender/blenloader/intern/versioning_270.c +++ b/source/blender/blenloader/intern/versioning_270.c @@ -290,7 +290,9 @@ void blo_do_versions_270(FileData *fd, Library *UNUSED(lib), Main *main) if (space_link->spacetype == SPACE_CLIP) { SpaceClip *space_clip = (SpaceClip *) space_link; if (space_clip->mode != SC_MODE_MASKEDIT) { - space_clip->mode = SC_MODE_TRACKING; + if (space_clip->mode != SC_MODE_TRACKING) { + space_clip->mode = SC_MODE_CORRESPONDENCE; + } } } } @@ -1609,6 +1611,33 @@ void blo_do_versions_270(FileData *fd, Library *UNUSED(lib), Main *main) } } } + + /* initialize regiondata for each SpaceClip, due to the newly brought RegionSpaceClip */ + if (!DNA_struct_elem_find(fd->filesdna, "SpaceClip", "MovieClip", "*secondary_clip")) { + for (bScreen *screen = main->screen.first; screen != NULL; screen = screen->id.next) { + for (ScrArea *sa = screen->areabase.first; sa != NULL; sa = sa->next) { + for (SpaceLink *sl = sa->spacedata.first; sl != NULL; sl = sl->next) { + if (sl->spacetype == SPACE_CLIP) { + ListBase *regionbase = (sl == sa->spacedata.first) ? &sa->regionbase : &sl->regionbase; + for (ARegion *ar = regionbase->first; ar != NULL; ar = ar->next) { + if (ar->regiontype == RGN_TYPE_WINDOW) { + SpaceClip *sc = (SpaceClip *)sl; + RegionSpaceClip *rsc = MEM_callocN(sizeof(RegionSpaceClip), "region data for clip"); + + rsc->xof = sc->xof; + rsc->yof = sc->yof; + rsc->xlockof = sc->xlockof; + rsc->ylockof = sc->ylockof; + rsc->zoom = sc->zoom; + rsc->flag = RSC_MAIN_CLIP; + ar->regiondata = rsc; + } + } + } + } + } + } + } } } diff --git a/source/blender/blenloader/intern/writefile.c b/source/blender/blenloader/intern/writefile.c index 57be237be6f..c71d9f3582c 100644 --- a/source/blender/blenloader/intern/writefile.c +++ b/source/blender/blenloader/intern/writefile.c @@ -2805,10 +2805,16 @@ static void write_region(WriteData *wd, ARegion *ar, int spacetype) } else - printf("regiondata write missing!\n"); + printf("spaceview3d regiondata write missing!\n"); + break; + case SPACE_CLIP: + if (ar->regiontype == RGN_TYPE_WINDOW) { + RegionSpaceClip *rsc = ar->regiondata; + writestruct(wd, DATA, RegionSpaceClip, 1, rsc); + } break; default: - printf("regiondata write missing!\n"); + printf("default regiondata write missing!\n"); } } } @@ -3286,6 +3292,14 @@ static void write_moviePlaneTracks(WriteData *wd, ListBase *plane_tracks_base) } } +static void write_movieCorrespondences(WriteData *wd, ListBase *correspondence_base) +{ + MovieTrackingCorrespondence *corr; + for (corr = correspondence_base->first; corr != NULL; corr = corr->next) { + writestruct(wd, DATA, MovieTrackingCorrespondence, 1, corr); + } +} + static void write_movieReconstruction(WriteData *wd, MovieTrackingReconstruction *reconstruction) { if (reconstruction->camnr) { @@ -3309,6 +3323,7 @@ static void write_movieclip(WriteData *wd, MovieClip *clip) write_movieTracks(wd, &tracking->tracks); write_moviePlaneTracks(wd, &tracking->plane_tracks); write_movieReconstruction(wd, &tracking->reconstruction); + write_movieCorrespondences(wd, &tracking->correspondences); object = tracking->objects.first; while (object) { diff --git a/source/blender/editors/include/ED_clip.h b/source/blender/editors/include/ED_clip.h index 5f8ebd87d19..fa28ddb5ad0 100644 --- a/source/blender/editors/include/ED_clip.h +++ b/source/blender/editors/include/ED_clip.h @@ -51,6 +51,7 @@ int ED_space_clip_view_clip_poll(struct bContext *C); int ED_space_clip_tracking_poll(struct bContext *C); int ED_space_clip_maskedit_poll(struct bContext *C); int ED_space_clip_maskedit_mask_poll(struct bContext *C); +int ED_space_clip_correspondence_poll(struct bContext *C); void ED_space_clip_get_size(struct SpaceClip *sc, int *width, int *height); void ED_space_clip_get_size_fl(struct SpaceClip *sc, float size[2]); @@ -60,8 +61,8 @@ void ED_space_clip_get_aspect_dimension_aware(struct SpaceClip *sc, float *aspx, int ED_space_clip_get_clip_frame_number(struct SpaceClip *sc); -struct ImBuf *ED_space_clip_get_buffer(struct SpaceClip *sc); -struct ImBuf *ED_space_clip_get_stable_buffer(struct SpaceClip *sc, float loc[2], float *scale, float *angle); +struct ImBuf *ED_space_clip_get_buffer(struct SpaceClip *sc, struct ARegion *ar); +struct ImBuf *ED_space_clip_get_stable_buffer(struct SpaceClip *sc, struct ARegion *ar, float loc[2], float *scale, float *angle); bool ED_space_clip_color_sample(struct Scene *scene, struct SpaceClip *sc, struct ARegion *ar, int mval[2], float r_col[3]); @@ -75,9 +76,16 @@ void ED_clip_mouse_pos(struct SpaceClip *sc, struct ARegion *ar, const int mval[ bool ED_space_clip_check_show_trackedit(struct SpaceClip *sc); bool ED_space_clip_check_show_maskedit(struct SpaceClip *sc); +bool ED_space_clip_check_show_correspondence(struct SpaceClip *sc); struct MovieClip *ED_space_clip_get_clip(struct SpaceClip *sc); +struct MovieClip *ED_space_clip_get_secondary_clip(struct SpaceClip *sc); +struct MovieClip *ED_space_clip_get_clip_in_region(struct SpaceClip *sc, struct ARegion *ar); void ED_space_clip_set_clip(struct bContext *C, struct bScreen *screen, struct SpaceClip *sc, struct MovieClip *clip); +void ED_space_clip_set_secondary_clip(struct bContext *C, struct bScreen *screen, struct SpaceClip *sc, struct MovieClip *secondary_clip); +void ED_clip_update_correspondence_mode(struct bContext *C, struct SpaceClip *sc); + +void ED_space_clip_region_set_lock_zero(struct bContext *C); struct Mask *ED_space_clip_get_mask(struct SpaceClip *sc); void ED_space_clip_set_mask(struct bContext *C, struct SpaceClip *sc, struct Mask *mask); diff --git a/source/blender/editors/include/UI_icons.h b/source/blender/editors/include/UI_icons.h index 8420591aa3e..b6d565e4e79 100644 --- a/source/blender/editors/include/UI_icons.h +++ b/source/blender/editors/include/UI_icons.h @@ -591,8 +591,8 @@ DEF_ICON(MOD_TRIANGULATE) DEF_ICON(MOD_WIREFRAME) DEF_ICON(MOD_DATA_TRANSFER) DEF_ICON(MOD_NORMALEDIT) +DEF_ICON(MOD_CORRESPONDENCE) #ifndef DEF_ICON_BLANK_SKIP - DEF_ICON(BLANK169) DEF_ICON(BLANK170) DEF_ICON(BLANK171) DEF_ICON(BLANK172) diff --git a/source/blender/editors/include/UI_resources.h b/source/blender/editors/include/UI_resources.h index f8a5f30a596..a5ea4e7ca9b 100644 --- a/source/blender/editors/include/UI_resources.h +++ b/source/blender/editors/include/UI_resources.h @@ -301,7 +301,10 @@ enum { TH_METADATA_TEXT, TH_EDGE_BEVEL, - TH_VERTEX_BEVEL + TH_VERTEX_BEVEL, + /* color theme for linked markers */ + TH_LINKED_MARKER, + TH_SEL_LINKED_MARKER }; /* XXX WARNING: previous is saved in file, so do not change order! */ diff --git a/source/blender/editors/interface/resources.c b/source/blender/editors/interface/resources.c index 539284030c2..689a7782783 100644 --- a/source/blender/editors/interface/resources.c +++ b/source/blender/editors/interface/resources.c @@ -597,6 +597,10 @@ const unsigned char *UI_ThemeGetColorPtr(bTheme *btheme, int spacetype, int colo cp = ts->act_marker; break; case TH_SEL_MARKER: cp = ts->sel_marker; break; + case TH_LINKED_MARKER: + cp = ts->linked_marker; break; + case TH_SEL_LINKED_MARKER: + cp = ts->sel_linked_marker; break; case TH_BUNDLE_SOLID: cp = ts->bundle_solid; break; case TH_DIS_MARKER: @@ -1196,6 +1200,8 @@ void ui_theme_init_default(void) rgba_char_args_set(btheme->tclip.act_marker, 0xff, 0xff, 0xff, 255); rgba_char_args_set(btheme->tclip.sel_marker, 0xff, 0xff, 0x00, 255); rgba_char_args_set(btheme->tclip.dis_marker, 0x7f, 0x00, 0x00, 255); + rgba_char_args_set(btheme->tclip.linked_marker, 0x85, 0xc1, 0xe9, 255); + rgba_char_args_set(btheme->tclip.sel_linked_marker, 0xcb, 0x43, 0x35, 255); rgba_char_args_set(btheme->tclip.lock_marker, 0x7f, 0x7f, 0x7f, 255); rgba_char_args_set(btheme->tclip.path_before, 0xff, 0x00, 0x00, 255); rgba_char_args_set(btheme->tclip.path_after, 0x00, 0x00, 0xff, 255); @@ -2739,6 +2745,9 @@ void init_userdef_do_versions(void) for (btheme = U.themes.first; btheme; btheme = btheme->next) { if (btheme->tact.keyframe_scale_fac < 0.1f) btheme->tact.keyframe_scale_fac = 1.0f; + + rgba_char_args_set(btheme->tclip.linked_marker, 0x85, 0xc1, 0xe9, 255); + rgba_char_args_set(btheme->tclip.sel_linked_marker, 0xcb, 0x43, 0x35, 255); } } diff --git a/source/blender/editors/space_clip/CMakeLists.txt b/source/blender/editors/space_clip/CMakeLists.txt index 32d48c9c564..d4e16f22b1b 100644 --- a/source/blender/editors/space_clip/CMakeLists.txt +++ b/source/blender/editors/space_clip/CMakeLists.txt @@ -53,6 +53,7 @@ set(SRC clip_utils.c space_clip.c tracking_ops.c + tracking_ops_correspondence.c tracking_ops_detect.c tracking_ops_orient.c tracking_ops_plane.c diff --git a/source/blender/editors/space_clip/clip_buttons.c b/source/blender/editors/space_clip/clip_buttons.c index 0378c68d12c..71bd2c31c41 100644 --- a/source/blender/editors/space_clip/clip_buttons.c +++ b/source/blender/editors/space_clip/clip_buttons.c @@ -105,8 +105,9 @@ void uiTemplateMovieClip(uiLayout *layout, bContext *C, PointerRNA *ptr, const c uiLayoutSetContextPointer(layout, "edit_movieclip", &clipptr); - if (!compact) + if (!compact) { uiTemplateID(layout, C, ptr, propname, NULL, "CLIP_OT_open", NULL); + } if (clip) { uiLayout *col; diff --git a/source/blender/editors/space_clip/clip_draw.c b/source/blender/editors/space_clip/clip_draw.c index 695d04d3850..78895502993 100644 --- a/source/blender/editors/space_clip/clip_draw.c +++ b/source/blender/editors/space_clip/clip_draw.c @@ -251,7 +251,7 @@ static void draw_movieclip_cache(SpaceClip *sc, ARegion *ar, MovieClip *clip, Sc static void draw_movieclip_notes(SpaceClip *sc, ARegion *ar) { - MovieClip *clip = ED_space_clip_get_clip(sc); + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); MovieTracking *tracking = &clip->tracking; char str[256] = {0}; bool full_redraw = false; @@ -285,7 +285,7 @@ static void draw_movieclip_muted(ARegion *ar, int width, int height, float zoomx static void draw_movieclip_buffer(const bContext *C, SpaceClip *sc, ARegion *ar, ImBuf *ibuf, int width, int height, float zoomx, float zoomy) { - MovieClip *clip = ED_space_clip_get_clip(sc); + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); int filter = GL_LINEAR; int x, y; @@ -328,7 +328,7 @@ static void draw_movieclip_buffer(const bContext *C, SpaceClip *sc, ARegion *ar, static void draw_stabilization_border(SpaceClip *sc, ARegion *ar, int width, int height, float zoomx, float zoomy) { int x, y; - MovieClip *clip = ED_space_clip_get_clip(sc); + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); /* find window pixel coordinates of origin */ UI_view2d_view_to_region(&ar->v2d, 0.0f, 0.0f, &x, &y); @@ -472,7 +472,7 @@ static void draw_track_path(SpaceClip *sc, MovieClip *UNUSED(clip), MovieTrackin glEnd(); } -static void draw_marker_outline(SpaceClip *sc, MovieTrackingTrack *track, MovieTrackingMarker *marker, +static void draw_marker_outline(SpaceClip *sc, ARegion *ar, MovieTrackingTrack *track, MovieTrackingMarker *marker, const float marker_pos[2], int width, int height) { int tiny = sc->flag & SC_SHOW_TINY_MARKER; @@ -481,8 +481,9 @@ static void draw_marker_outline(SpaceClip *sc, MovieTrackingTrack *track, MovieT UI_ThemeColor(TH_MARKER_OUTLINE); - px[0] = 1.0f / width / sc->zoom; - px[1] = 1.0f / height / sc->zoom; + RegionSpaceClip *rsc = (RegionSpaceClip*) ar->regiondata; + px[0] = 1.0f / width / rsc->zoom; + px[1] = 1.0f / height / rsc->zoom; glLineWidth(tiny ? 1.0f : 3.0f); @@ -547,7 +548,22 @@ static void draw_marker_outline(SpaceClip *sc, MovieTrackingTrack *track, MovieT glPopMatrix(); } -static void track_colors(MovieTrackingTrack *track, int act, float col[3], float scol[3]) +/* return whether the track is a linked track by iterating the correspondence base in tracking. */ +static bool is_track_linked(MovieTracking *tracking, MovieClip *clip, MovieTrackingTrack *track) +{ + MovieTrackingCorrespondence *corr = tracking->correspondences.first; + while (corr) { + if ((corr->self_clip == clip && strcmp(corr->self_track_name, track->name) == 0) || + (corr->other_clip == clip && strcmp(corr->other_track_name, track->name) == 0)) + { + return true; + } + corr = corr->next; + } + return false; +} + +static void track_colors(MovieTrackingTrack *track, int act, int link, float col[3], float scol[3]) { if (track->flag & TRACK_CUSTOMCOLOR) { if (act) @@ -558,26 +574,36 @@ static void track_colors(MovieTrackingTrack *track, int act, float col[3], float mul_v3_v3fl(col, track->color, 0.5f); } else { - UI_GetThemeColor3fv(TH_MARKER, col); + if (link) + UI_GetThemeColor3fv(TH_LINKED_MARKER, col); + else + UI_GetThemeColor3fv(TH_MARKER, col); if (act) UI_GetThemeColor3fv(TH_ACT_MARKER, scol); else - UI_GetThemeColor3fv(TH_SEL_MARKER, scol); + if (link) + UI_GetThemeColor3fv(TH_SEL_LINKED_MARKER, scol); + else + UI_GetThemeColor3fv(TH_SEL_MARKER, scol); } } -static void draw_marker_areas(SpaceClip *sc, MovieTrackingTrack *track, MovieTrackingMarker *marker, +static void draw_marker_areas(SpaceClip *sc, ARegion *ar, MovieTrackingTrack *track, MovieTrackingMarker *marker, const float marker_pos[2], int width, int height, int act, int sel) { int tiny = sc->flag & SC_SHOW_TINY_MARKER; bool show_search = false; float col[3], scol[3], px[2]; - track_colors(track, act, col, scol); + MovieClip *mc = ED_space_clip_get_clip_in_region(sc, ar); + MovieTracking *tracking = &mc->tracking; + bool link = is_track_linked(tracking, mc, track); + track_colors(track, act, link, col, scol); - px[0] = 1.0f / width / sc->zoom; - px[1] = 1.0f / height / sc->zoom; + RegionSpaceClip *rsc = (RegionSpaceClip*) ar->regiondata; + px[0] = 1.0f / width / rsc->zoom; + px[1] = 1.0f / height / rsc->zoom; glLineWidth(1.0f); @@ -785,7 +811,7 @@ static void draw_marker_slide_triangle(float x, float y, float dx, float dy, int glEnd(); } -static void draw_marker_slide_zones(SpaceClip *sc, MovieTrackingTrack *track, MovieTrackingMarker *marker, +static void draw_marker_slide_zones(SpaceClip *sc, ARegion *ar, MovieTrackingTrack *track, MovieTrackingMarker *marker, const float marker_pos[2], int outline, int sel, int act, int width, int height) { float dx, dy, patdx, patdy, searchdx, searchdy; @@ -798,7 +824,10 @@ static void draw_marker_slide_zones(SpaceClip *sc, MovieTrackingTrack *track, Mo if (!TRACK_VIEW_SELECTED(sc, track) || track->flag & TRACK_LOCKED) return; - track_colors(track, act, col, scol); + MovieClip *mc = ED_space_clip_get_clip_in_region(sc, ar); + MovieTracking *tracking = &mc->tracking; + bool link = is_track_linked(tracking, mc, track); + track_colors(track, act, link, col, scol); if (outline) { UI_ThemeColor(TH_MARKER_OUTLINE); @@ -807,8 +836,9 @@ static void draw_marker_slide_zones(SpaceClip *sc, MovieTrackingTrack *track, Mo glPushMatrix(); glTranslate2fv(marker_pos); - dx = 6.0f / width / sc->zoom; - dy = 6.0f / height / sc->zoom; + RegionSpaceClip *rsc = (RegionSpaceClip*) ar->regiondata; + dx = 6.0f / width / rsc->zoom; + dy = 6.0f / height / rsc->zoom; side = get_shortest_pattern_side(marker); patdx = min_ff(dx * 2.0f / 3.0f, side / 6.0f) * UI_DPI_FAC; @@ -817,8 +847,8 @@ static void draw_marker_slide_zones(SpaceClip *sc, MovieTrackingTrack *track, Mo searchdx = min_ff(dx, (marker->search_max[0] - marker->search_min[0]) / 6.0f) * UI_DPI_FAC; searchdy = min_ff(dy, (marker->search_max[1] - marker->search_min[1]) / 6.0f) * UI_DPI_FAC; - px[0] = 1.0f / sc->zoom / width / sc->scale; - px[1] = 1.0f / sc->zoom / height / sc->scale; + px[0] = 1.0f / rsc->zoom / width / sc->scale; + px[1] = 1.0f / rsc->zoom / height / sc->scale; if ((sc->flag & SC_SHOW_MARKER_SEARCH) && ((track->search_flag & SELECT) == sel || outline)) { if (!outline) { @@ -1099,7 +1129,7 @@ static void draw_plane_marker_image(Scene *scene, BKE_image_release_ibuf(image, ibuf, lock); } -static void draw_plane_marker_ex(SpaceClip *sc, Scene *scene, MovieTrackingPlaneTrack *plane_track, +static void draw_plane_marker_ex(SpaceClip *sc, ARegion *ar, Scene *scene, MovieTrackingPlaneTrack *plane_track, MovieTrackingPlaneMarker *plane_marker, bool is_active_track, bool draw_outline, int width, int height) { @@ -1124,8 +1154,9 @@ static void draw_plane_marker_ex(SpaceClip *sc, Scene *scene, MovieTrackingPlane } } - px[0] = 1.0f / width / sc->zoom; - px[1] = 1.0f / height / sc->zoom; + RegionSpaceClip *rsc = (RegionSpaceClip*) ar->regiondata; + px[0] = 1.0f / width / rsc->zoom; + px[1] = 1.0f / height / rsc->zoom; /* Draw image */ if (draw_outline == false) { @@ -1162,12 +1193,12 @@ static void draw_plane_marker_ex(SpaceClip *sc, Scene *scene, MovieTrackingPlane glBegin(GL_LINES); - getArrowEndPoint(width, height, sc->zoom, plane_marker->corners[0], plane_marker->corners[1], end_point); + getArrowEndPoint(width, height, rsc->zoom, plane_marker->corners[0], plane_marker->corners[1], end_point); glColor3f(1.0, 0.0, 0.0f); glVertex2fv(plane_marker->corners[0]); glVertex2fv(end_point); - getArrowEndPoint(width, height, sc->zoom, plane_marker->corners[0], plane_marker->corners[3], end_point); + getArrowEndPoint(width, height, rsc->zoom, plane_marker->corners[0], plane_marker->corners[3], end_point); glColor3f(0.0, 1.0, 0.0f); glVertex2fv(plane_marker->corners[0]); glVertex2fv(end_point); @@ -1195,28 +1226,28 @@ static void draw_plane_marker_ex(SpaceClip *sc, Scene *scene, MovieTrackingPlane } } -static void draw_plane_marker_outline(SpaceClip *sc, Scene *scene, MovieTrackingPlaneTrack *plane_track, +static void draw_plane_marker_outline(SpaceClip *sc, ARegion *ar, Scene *scene, MovieTrackingPlaneTrack *plane_track, MovieTrackingPlaneMarker *plane_marker, int width, int height) { - draw_plane_marker_ex(sc, scene, plane_track, plane_marker, false, true, width, height); + draw_plane_marker_ex(sc, ar, scene, plane_track, plane_marker, false, true, width, height); } -static void draw_plane_marker(SpaceClip *sc, Scene *scene, MovieTrackingPlaneTrack *plane_track, +static void draw_plane_marker(SpaceClip *sc, ARegion *ar, Scene *scene, MovieTrackingPlaneTrack *plane_track, MovieTrackingPlaneMarker *plane_marker, bool is_active_track, int width, int height) { - draw_plane_marker_ex(sc, scene, plane_track, plane_marker, is_active_track, false, width, height); + draw_plane_marker_ex(sc, ar, scene, plane_track, plane_marker, is_active_track, false, width, height); } -static void draw_plane_track(SpaceClip *sc, Scene *scene, MovieTrackingPlaneTrack *plane_track, +static void draw_plane_track(SpaceClip *sc, ARegion *ar, Scene *scene, MovieTrackingPlaneTrack *plane_track, int framenr, bool is_active_track, int width, int height) { MovieTrackingPlaneMarker *plane_marker; plane_marker = BKE_tracking_plane_marker_get(plane_track, framenr); - draw_plane_marker_outline(sc, scene, plane_track, plane_marker, width, height); - draw_plane_marker(sc, scene, plane_track, plane_marker, is_active_track, width, height); + draw_plane_marker_outline(sc, ar, scene, plane_track, plane_marker, width, height); + draw_plane_marker(sc, ar, scene, plane_track, plane_marker, is_active_track, width, height); } /* Draw all kind of tracks. */ @@ -1260,7 +1291,7 @@ static void draw_tracking_tracks(SpaceClip *sc, Scene *scene, ARegion *ar, Movie plane_track = plane_track->next) { if ((plane_track->flag & PLANE_TRACK_HIDDEN) == 0) { - draw_plane_track(sc, scene, plane_track, framenr, plane_track == active_plane_track, width, height); + draw_plane_track(sc, ar, scene, plane_track, framenr, plane_track == active_plane_track, width, height); } } @@ -1325,10 +1356,10 @@ static void draw_tracking_tracks(SpaceClip *sc, Scene *scene, ARegion *ar, Movie if (MARKER_VISIBLE(sc, track, marker)) { copy_v2_v2(cur_pos, fp ? fp : marker->pos); - draw_marker_outline(sc, track, marker, cur_pos, width, height); - draw_marker_areas(sc, track, marker, cur_pos, width, height, 0, 0); - draw_marker_slide_zones(sc, track, marker, cur_pos, 1, 0, 0, width, height); - draw_marker_slide_zones(sc, track, marker, cur_pos, 0, 0, 0, width, height); + draw_marker_outline(sc, ar, track, marker, cur_pos, width, height); + draw_marker_areas(sc, ar, track, marker, cur_pos, width, height, 0, 0); + draw_marker_slide_zones(sc, ar, track, marker, cur_pos, 1, 0, 0, width, height); + draw_marker_slide_zones(sc, ar, track, marker, cur_pos, 0, 0, 0, width, height); if (fp) fp += 2; @@ -1351,8 +1382,8 @@ static void draw_tracking_tracks(SpaceClip *sc, Scene *scene, ARegion *ar, Movie if (!act) { copy_v2_v2(cur_pos, fp ? fp : marker->pos); - draw_marker_areas(sc, track, marker, cur_pos, width, height, 0, 1); - draw_marker_slide_zones(sc, track, marker, cur_pos, 0, 1, 0, width, height); + draw_marker_areas(sc, ar, track, marker, cur_pos, width, height, 0, 1); + draw_marker_slide_zones(sc, ar, track, marker, cur_pos, 0, 1, 0, width, height); } if (fp) @@ -1371,8 +1402,8 @@ static void draw_tracking_tracks(SpaceClip *sc, Scene *scene, ARegion *ar, Movie if (MARKER_VISIBLE(sc, act_track, marker)) { copy_v2_v2(cur_pos, active_pos ? active_pos : marker->pos); - draw_marker_areas(sc, act_track, marker, cur_pos, width, height, 1, 1); - draw_marker_slide_zones(sc, act_track, marker, cur_pos, 0, 1, 1, width, height); + draw_marker_areas(sc, ar, act_track, marker, cur_pos, width, height, 1, 1); + draw_marker_slide_zones(sc, ar, act_track, marker, cur_pos, 0, 1, 1, width, height); } } } @@ -1658,7 +1689,7 @@ static void draw_distortion(SpaceClip *sc, ARegion *ar, MovieClip *clip, void clip_draw_main(const bContext *C, SpaceClip *sc, ARegion *ar) { - MovieClip *clip = ED_space_clip_get_clip(sc); + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); Scene *scene = CTX_data_scene(C); ImBuf *ibuf = NULL; int width, height; @@ -1679,7 +1710,7 @@ void clip_draw_main(const bContext *C, SpaceClip *sc, ARegion *ar) float smat[4][4], ismat[4][4]; if ((sc->flag & SC_MUTE_FOOTAGE) == 0) { - ibuf = ED_space_clip_get_stable_buffer(sc, sc->loc, + ibuf = ED_space_clip_get_stable_buffer(sc, ar, sc->loc, &sc->scale, &sc->angle); } @@ -1699,7 +1730,7 @@ void clip_draw_main(const bContext *C, SpaceClip *sc, ARegion *ar) mul_m4_series(sc->unistabmat, smat, sc->stabmat, ismat); } else if ((sc->flag & SC_MUTE_FOOTAGE) == 0) { - ibuf = ED_space_clip_get_buffer(sc); + ibuf = ED_space_clip_get_buffer(sc, ar); zero_v2(sc->loc); sc->scale = 1.0f; @@ -1728,7 +1759,7 @@ void clip_draw_main(const bContext *C, SpaceClip *sc, ARegion *ar) void clip_draw_cache_and_notes(const bContext *C, SpaceClip *sc, ARegion *ar) { Scene *scene = CTX_data_scene(C); - MovieClip *clip = ED_space_clip_get_clip(sc); + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); if (clip) { draw_movieclip_cache(sc, ar, clip, scene); draw_movieclip_notes(sc, ar); @@ -1739,7 +1770,8 @@ void clip_draw_cache_and_notes(const bContext *C, SpaceClip *sc, ARegion *ar) void clip_draw_grease_pencil(bContext *C, int onlyv2d) { SpaceClip *sc = CTX_wm_space_clip(C); - MovieClip *clip = ED_space_clip_get_clip(sc); + ARegion *ar = CTX_wm_region(C); + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); if (!clip) return; diff --git a/source/blender/editors/space_clip/clip_editor.c b/source/blender/editors/space_clip/clip_editor.c index 14d0f909d23..83d001923e2 100644 --- a/source/blender/editors/space_clip/clip_editor.c +++ b/source/blender/editors/space_clip/clip_editor.c @@ -49,6 +49,7 @@ #include "BLI_math.h" #include "BLI_rect.h" #include "BLI_task.h" +#include "BLI_listbase.h" #include "BKE_global.h" #include "BKE_main.h" @@ -57,7 +58,7 @@ #include "BKE_context.h" #include "BKE_tracking.h" #include "BKE_library.h" - +#include "BKE_screen.h" #include "IMB_colormanagement.h" #include "IMB_imbuf_types.h" @@ -133,6 +134,17 @@ int ED_space_clip_maskedit_mask_poll(bContext *C) return false; } +int ED_space_clip_correspondence_poll(bContext *C) +{ + SpaceClip *sc = CTX_wm_space_clip(C); + + if (sc && sc->clip) { + return ED_space_clip_check_show_correspondence(sc); + } + + return false; +} + /* ******** common editing functions ******** */ void ED_space_clip_get_size(SpaceClip *sc, int *width, int *height) @@ -225,12 +237,13 @@ int ED_space_clip_get_clip_frame_number(SpaceClip *sc) return BKE_movieclip_remap_scene_to_clip_frame(clip, sc->user.framenr); } -ImBuf *ED_space_clip_get_buffer(SpaceClip *sc) +ImBuf *ED_space_clip_get_buffer(SpaceClip *sc, ARegion *ar) { - if (sc->clip) { + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); + if (clip) { ImBuf *ibuf; - ibuf = BKE_movieclip_get_postprocessed_ibuf(sc->clip, &sc->user, sc->postproc_flag); + ibuf = BKE_movieclip_get_postprocessed_ibuf(clip, &sc->user, sc->postproc_flag); if (ibuf && (ibuf->rect || ibuf->rect_float)) return ibuf; @@ -242,12 +255,13 @@ ImBuf *ED_space_clip_get_buffer(SpaceClip *sc) return NULL; } -ImBuf *ED_space_clip_get_stable_buffer(SpaceClip *sc, float loc[2], float *scale, float *angle) +ImBuf *ED_space_clip_get_stable_buffer(SpaceClip *sc, ARegion *ar, float loc[2], float *scale, float *angle) { - if (sc->clip) { + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); + if (clip) { ImBuf *ibuf; - ibuf = BKE_movieclip_get_stable_ibuf(sc->clip, &sc->user, loc, scale, angle, sc->postproc_flag); + ibuf = BKE_movieclip_get_stable_ibuf(clip, &sc->user, loc, scale, angle, sc->postproc_flag); if (ibuf && (ibuf->rect || ibuf->rect_float)) return ibuf; @@ -268,7 +282,7 @@ bool ED_space_clip_color_sample(Scene *scene, SpaceClip *sc, ARegion *ar, int mv float fx, fy, co[2]; bool ret = false; - ibuf = ED_space_clip_get_buffer(sc); + ibuf = ED_space_clip_get_buffer(sc, ar); if (!ibuf) { return false; } @@ -406,6 +420,7 @@ static bool selected_boundbox(const bContext *C, float min[2], float max[2]) bool ED_clip_view_selection(const bContext *C, ARegion *ar, bool fit) { SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = (RegionSpaceClip*) ar->regiondata; int w, h, frame_width, frame_height; float min[2], max[2]; @@ -418,8 +433,8 @@ bool ED_clip_view_selection(const bContext *C, ARegion *ar, bool fit) return false; /* center view */ - clip_view_center_to_point(sc, (max[0] + min[0]) / (2 * frame_width), - (max[1] + min[1]) / (2 * frame_height)); + clip_view_center_to_point(sc, rsc, (max[0] + min[0]) / (2 * frame_width), + (max[1] + min[1]) / (2 * frame_height)); w = max[0] - min[0]; h = max[1] - min[1]; @@ -439,8 +454,8 @@ bool ED_clip_view_selection(const bContext *C, ARegion *ar, bool fit) newzoom = 1.0f / power_of_2(1.0f / min_ff(zoomx, zoomy)); - if (fit || sc->zoom > newzoom) - sc->zoom = newzoom; + if (fit || rsc->zoom > newzoom) + rsc->zoom = newzoom; } return true; @@ -549,6 +564,15 @@ bool ED_space_clip_check_show_maskedit(SpaceClip *sc) return false; } +bool ED_space_clip_check_show_correspondence(SpaceClip *sc) +{ + if (sc) { + return sc->mode == SC_MODE_CORRESPONDENCE; + } + + return false; +} + /* ******** clip editing functions ******** */ MovieClip *ED_space_clip_get_clip(SpaceClip *sc) @@ -556,6 +580,27 @@ MovieClip *ED_space_clip_get_clip(SpaceClip *sc) return sc->clip; } +MovieClip *ED_space_clip_get_secondary_clip(SpaceClip *sc) +{ + return sc->secondary_clip; +} + +MovieClip *ED_space_clip_get_clip_in_region(SpaceClip *sc, ARegion *ar) +{ + RegionSpaceClip *rsc = ar->regiondata; + + /* dopsheet and graph don't have regiondata, so just return the main clip */ + if (!rsc) { + return ED_space_clip_get_clip(sc); + } + else if (rsc->flag == RSC_MAIN_CLIP) { + return ED_space_clip_get_clip(sc); + } + else { //rsc->flag == RSC_SECONDARY_CLIP + return ED_space_clip_get_secondary_clip(sc); + } +} + void ED_space_clip_set_clip(bContext *C, bScreen *screen, SpaceClip *sc, MovieClip *clip) { MovieClip *old_clip; @@ -603,6 +648,161 @@ void ED_space_clip_set_clip(bContext *C, bScreen *screen, SpaceClip *sc, MovieCl WM_event_add_notifier(C, NC_MOVIECLIP | NA_SELECTED, sc->clip); } +void ED_space_clip_set_secondary_clip(bContext *C, bScreen *screen, SpaceClip *sc, MovieClip *secondary_clip) +{ + MovieClip *old_clip; + bool old_clip_visible = false; + + if (!screen && C) + screen = CTX_wm_screen(C); + + old_clip = sc->secondary_clip; + sc->secondary_clip = secondary_clip; + + id_us_ensure_real((ID *)sc->secondary_clip); + + if (screen && sc->view == SC_VIEW_CLIP) { + ScrArea *area; + SpaceLink *sl; + + for (area = screen->areabase.first; area; area = area->next) { + for (sl = area->spacedata.first; sl; sl = sl->next) { + if (sl->spacetype == SPACE_CLIP) { + SpaceClip *cur_sc = (SpaceClip *) sl; + + if (cur_sc != sc) { + if (cur_sc->view == SC_VIEW_CLIP) { + if (cur_sc->secondary_clip == old_clip) + old_clip_visible = true; + } + else { + if (cur_sc->secondary_clip == old_clip || cur_sc->secondary_clip == NULL) { + cur_sc->secondary_clip = secondary_clip; + } + } + } + } + } + } + } + + /* If clip is no longer visible on screen, free memory used by it's cache */ + if (old_clip && old_clip != secondary_clip && !old_clip_visible) { + BKE_movieclip_clear_cache(old_clip); + } + + if (C) + WM_event_add_notifier(C, NC_MOVIECLIP | NA_SELECTED, sc->secondary_clip); +} + +/* ******** split view when changing to correspondence mode ******** */ + +static void region_splitview_init(ScrArea *sa, ARegion *ar, SpaceClip *sc, eRegionSpaceClip_Flag flag) +{ + /* set the region type so that clip_main_region_draw is aware of this */ + RegionSpaceClip *rsc = ar->regiondata; + rsc->flag = flag; + + /* XXX: Hack to make proper alignment decisions made in + * region_rect_recursive(). + * + * This is quite bad and should ideally be addressed by the layout + * management which currently check whether it is enough space to fit both + * regions based on their current size. + * + * What we want instead happening in the layout engine is that the regions + * will pr properly scaled so they all fit and use the whole available + * space. Just similar to what's happening with quad split. + * + * However, it is a can of worms on it's own, so for now we just scale the + * regions in a way that they pass fitting tests in region_rect_recursive(). + * + * TODO(sergey): Need to check this with the interface department! + */ + ar->sizey /= 2; +} + +void ED_clip_update_correspondence_mode(bContext *C, SpaceClip *sc) +{ + /* search forward and backward to find the drawing region */ + ARegion *ar_origin = CTX_wm_region(C); + bool find_draw_region = false; + ARegion *ar = ar_origin; + while (ar && !find_draw_region) { + if (ar->regiontype == RGN_TYPE_WINDOW) { + find_draw_region = true; + break; + } + ar = ar->prev; + } + /* recover to the current ARegion and search backwards*/ + ar = ar_origin; + while (ar && !find_draw_region) { + if (ar->regiontype == RGN_TYPE_WINDOW) { + find_draw_region = true; + break; + } + ar = ar->next; + } + BLI_assert(find_draw_region == true && ar != NULL); + + /* some rules related to changing between correspondence mode and other mode*/ + if (ar->regiontype != RGN_TYPE_WINDOW) { + return; + } + else if (sc->mode != SC_MODE_CORRESPONDENCE && ar->alignment == RGN_ALIGN_VSPLIT) { + ///* Exit split-view */ + ScrArea *sa = CTX_wm_area(C); + ARegion *arn; + + /* keep current region */ + ar->alignment = 0; + + for (ar = sa->regionbase.first; ar; ar = arn) { + arn = ar->next; + if (ar->alignment == RGN_ALIGN_VSPLIT) { + ED_region_exit(C, ar); + BKE_area_region_free(sa->type, ar); + BLI_remlink(&sa->regionbase, ar); + MEM_freeN(ar); + } + } + ED_area_tag_redraw(sa); + WM_event_add_notifier(C, NC_SCREEN | NA_EDITED, NULL); + } + else if (ar->next) { + return; // Only last region can be splitted + } + else if (sc->mode == SC_MODE_CORRESPONDENCE) { + /* Enter split-view */ + ScrArea *sa = CTX_wm_area(C); + + ar->alignment = RGN_ALIGN_VSPLIT; + + /* copy the current ar */ + ARegion *newar = BKE_area_region_copy(sa->type, ar); + BLI_addtail(&sa->regionbase, newar); + + /* update split view */ + if (sa->spacetype == SPACE_CLIP) { + region_splitview_init(sa, ar, sc, RSC_MAIN_CLIP); + region_splitview_init(sa, (ar = ar->next), sc, RSC_SECONDARY_CLIP); + } + ED_area_tag_redraw(sa); + WM_event_add_notifier(C, NC_SCREEN | NA_EDITED, NULL); + } +} + +/* ******** space clip region functions ******** */ + +void ED_space_clip_region_set_lock_zero(bContext *C) +{ + RegionSpaceClip *rsc = CTX_wm_region_clip(C); + + rsc->xlockof = 0.f; + rsc->ylockof = 0.f; +} + /* ******** masking editing functions ******** */ Mask *ED_space_clip_get_mask(SpaceClip *sc) diff --git a/source/blender/editors/space_clip/clip_intern.h b/source/blender/editors/space_clip/clip_intern.h index 14393c6968b..a5fc7706653 100644 --- a/source/blender/editors/space_clip/clip_intern.h +++ b/source/blender/editors/space_clip/clip_intern.h @@ -40,6 +40,7 @@ struct MovieTrackingTrack; struct Scene; struct ScrArea; struct SpaceClip; +struct RegionSpaceClip; struct wmOperatorType; /* channel heights */ @@ -94,6 +95,7 @@ void CLIP_OT_graph_disable_markers(struct wmOperatorType *ot); /* clip_ops.c */ void CLIP_OT_open(struct wmOperatorType *ot); +void CLIP_OT_open_secondary(struct wmOperatorType *ot); void CLIP_OT_reload(struct wmOperatorType *ot); void CLIP_OT_view_pan(struct wmOperatorType *ot); void CLIP_OT_view_zoom(wmOperatorType *ot); @@ -139,7 +141,7 @@ void clip_graph_tracking_iterate(struct SpaceClip *sc, bool selected_only, bool void clip_delete_track(struct bContext *C, struct MovieClip *clip, struct MovieTrackingTrack *track); void clip_delete_marker(struct bContext *C, struct MovieClip *clip, struct MovieTrackingTrack *track, struct MovieTrackingMarker *marker); -void clip_view_center_to_point(SpaceClip *sc, float x, float y); +void clip_view_center_to_point(SpaceClip *sc, RegionSpaceClip *rsc, float x, float y); void clip_draw_cfra(struct SpaceClip *sc, struct ARegion *ar, struct Scene *scene); void clip_draw_sfra_efra(struct View2D *v2d, struct Scene *scene); @@ -206,6 +208,11 @@ void CLIP_OT_slide_plane_marker(struct wmOperatorType *ot); void CLIP_OT_keyframe_insert(struct wmOperatorType *ot); void CLIP_OT_keyframe_delete(struct wmOperatorType *ot); +/* tracking_ops_correspondence */ +void CLIP_OT_add_correspondence(wmOperatorType *ot); +void CLIP_OT_delete_correspondence(wmOperatorType *ot); +void CLIP_OT_solve_multiview(wmOperatorType *ot); + /* tracking_select.c */ void CLIP_OT_select(struct wmOperatorType *ot); void CLIP_OT_select_all(struct wmOperatorType *ot); diff --git a/source/blender/editors/space_clip/clip_ops.c b/source/blender/editors/space_clip/clip_ops.c index 9430ee626ba..7d60346bf76 100644 --- a/source/blender/editors/space_clip/clip_ops.c +++ b/source/blender/editors/space_clip/clip_ops.c @@ -91,24 +91,25 @@ static void sclip_zoom_set(const bContext *C, float zoom, float location[2]) SpaceClip *sc = CTX_wm_space_clip(C); ARegion *ar = CTX_wm_region(C); - float oldzoom = sc->zoom; + RegionSpaceClip *rsc = CTX_wm_region_clip(C); + float oldzoom = rsc->zoom; int width, height; - sc->zoom = zoom; + rsc->zoom = zoom; - if (sc->zoom < 0.1f || sc->zoom > 4.0f) { + if (rsc->zoom < 0.1f || rsc->zoom > 4.0f) { /* check zoom limits */ ED_space_clip_get_size(sc, &width, &height); - width *= sc->zoom; - height *= sc->zoom; + width *= rsc->zoom; + height *= rsc->zoom; - if ((width < 4) && (height < 4) && sc->zoom < oldzoom) - sc->zoom = oldzoom; - else if (BLI_rcti_size_x(&ar->winrct) <= sc->zoom) - sc->zoom = oldzoom; - else if (BLI_rcti_size_y(&ar->winrct) <= sc->zoom) - sc->zoom = oldzoom; + if ((width < 4) && (height < 4)) + rsc->zoom = oldzoom; + else if (BLI_rcti_size_x(&ar->winrct) <= rsc->zoom) + rsc->zoom = oldzoom; + else if (BLI_rcti_size_y(&ar->winrct) <= rsc->zoom) + rsc->zoom = oldzoom; } if ((U.uiflag & USER_ZOOM_TO_MOUSEPOS) && location) { @@ -116,25 +117,25 @@ static void sclip_zoom_set(const bContext *C, float zoom, float location[2]) ED_space_clip_get_size(sc, &width, &height); - dx = ((location[0] - 0.5f) * width - sc->xof) * (sc->zoom - oldzoom) / sc->zoom; - dy = ((location[1] - 0.5f) * height - sc->yof) * (sc->zoom - oldzoom) / sc->zoom; + dx = ((location[0] - 0.5f) * width - rsc->xof) * (rsc->zoom - oldzoom) / rsc->zoom; + dy = ((location[1] - 0.5f) * height - rsc->yof) * (rsc->zoom - oldzoom) / rsc->zoom; if (sc->flag & SC_LOCK_SELECTION) { - sc->xlockof += dx; - sc->ylockof += dy; + rsc->xlockof += dx; + rsc->ylockof += dy; } else { - sc->xof += dx; - sc->yof += dy; + rsc->xof += dx; + rsc->yof += dy; } } } static void sclip_zoom_set_factor(const bContext *C, float zoomfac, float location[2]) { - SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_data(C); - sclip_zoom_set(C, sc->zoom * zoomfac, location); + sclip_zoom_set(C, rsc->zoom * zoomfac, location); } static void sclip_zoom_set_factor_exec(bContext *C, const wmEvent *event, float factor) @@ -305,6 +306,133 @@ void CLIP_OT_open(wmOperatorType *ot) WM_FILESEL_RELPATH | WM_FILESEL_FILES | WM_FILESEL_DIRECTORY, FILE_DEFAULTDISPLAY, FILE_SORT_ALPHA); } +/******************** open a secondary clip in Correspondence mode operator ********************/ +static int open_secondary_exec(bContext *C, wmOperator *op) +{ + SpaceClip *sc = CTX_wm_space_clip(C); + bScreen *screen = CTX_wm_screen(C); + Main *bmain = CTX_data_main(C); + PropertyPointerRNA *pprop; + PointerRNA idptr; + MovieClip *clip = NULL; + char str[FILE_MAX]; + + if (RNA_collection_length(op->ptr, "files")) { + PointerRNA fileptr; + PropertyRNA *prop; + char dir_only[FILE_MAX], file_only[FILE_MAX]; + bool relative = RNA_boolean_get(op->ptr, "relative_path"); + + RNA_string_get(op->ptr, "directory", dir_only); + if (relative) + BLI_path_rel(dir_only, G.main->name); + + prop = RNA_struct_find_property(op->ptr, "files"); + RNA_property_collection_lookup_int(op->ptr, prop, 0, &fileptr); + RNA_string_get(&fileptr, "name", file_only); + + BLI_join_dirfile(str, sizeof(str), dir_only, file_only); + } + else { + BKE_report(op->reports, RPT_ERROR, "No files selected to be opened"); + + return OPERATOR_CANCELLED; + } + + /* default to frame 1 if there's no scene in context */ + + errno = 0; + + clip = BKE_movieclip_file_add_exists(bmain, str); + + if (!clip) { + if (op->customdata) + MEM_freeN(op->customdata); + + BKE_reportf(op->reports, RPT_ERROR, "Cannot read '%s': %s", str, + errno ? strerror(errno) : TIP_("unsupported movie clip format")); + + return OPERATOR_CANCELLED; + } + + if (!op->customdata) + open_init(C, op); + + /* hook into UI */ + pprop = op->customdata; + + if (pprop->prop) { + /* when creating new ID blocks, use is already 1, but RNA + * pointer se also increases user, so this compensates it */ + id_us_min(&clip->id); + + RNA_id_pointer_create(&clip->id, &idptr); + RNA_property_pointer_set(&pprop->ptr, pprop->prop, idptr); + RNA_property_update(C, &pprop->ptr, pprop->prop); + } + else if (sc) { + ED_space_clip_set_secondary_clip(C, screen, sc, clip); + } + + WM_event_add_notifier(C, NC_MOVIECLIP | NA_ADDED, clip); + + MEM_freeN(op->customdata); + + return OPERATOR_FINISHED; +} + +static int open_secondary_invoke(bContext *C, wmOperator *op, const wmEvent *UNUSED(event)) +{ + SpaceClip *sc = CTX_wm_space_clip(C); + char path[FILE_MAX]; + MovieClip *secondary_clip = NULL; + + if (sc) + secondary_clip = ED_space_clip_get_secondary_clip(sc); + + if (secondary_clip) { + BLI_strncpy(path, secondary_clip->name, sizeof(path)); + + BLI_path_abs(path, G.main->name); + BLI_parent_dir(path); + } + else { + BLI_strncpy(path, U.textudir, sizeof(path)); + } + + if (RNA_struct_property_is_set(op->ptr, "files")) + return open_secondary_exec(C, op); + + if (!RNA_struct_property_is_set(op->ptr, "relative_path")) + RNA_boolean_set(op->ptr, "relative_path", (U.flag & USER_RELPATHS) != 0); + + open_init(C, op); + + clip_filesel(C, op, path); + + return OPERATOR_RUNNING_MODAL; +} + +void CLIP_OT_open_secondary(wmOperatorType *ot) +{ + /* identifiers */ + ot->name = "Open Secondary Clip"; + ot->description = "Load a sequence of frames or a movie file in correspondence mode"; + ot->idname = "CLIP_OT_open_secondary"; + + /* api callbacks */ + ot->exec = open_secondary_exec; + ot->invoke = open_secondary_invoke; + ot->cancel = open_cancel; + + /* flags */ + ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO; + + /* properties */ + WM_operator_properties_filesel( + ot, FILE_TYPE_FOLDER | FILE_TYPE_IMAGE | FILE_TYPE_MOVIE, FILE_SPECIAL, FILE_OPENFILE, + WM_FILESEL_RELPATH | WM_FILESEL_FILES | WM_FILESEL_DIRECTORY, FILE_DEFAULTDISPLAY, FILE_SORT_ALPHA); +} /******************* reload clip operator *********************/ static int reload_exec(bContext *C, wmOperator *UNUSED(op)) @@ -344,6 +472,7 @@ typedef struct ViewPanData { static void view_pan_init(bContext *C, wmOperator *op, const wmEvent *event) { SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); ViewPanData *vpd; op->customdata = vpd = MEM_callocN(sizeof(ViewPanData), "ClipViewPanData"); @@ -353,9 +482,9 @@ static void view_pan_init(bContext *C, wmOperator *op, const wmEvent *event) vpd->y = event->y; if (sc->flag & SC_LOCK_SELECTION) - vpd->vec = &sc->xlockof; + vpd->vec = &rsc->xlockof; else - vpd->vec = &sc->xof; + vpd->vec = &rsc->xof; copy_v2_v2(&vpd->xof, vpd->vec); copy_v2_v2(&vpd->xorig, &vpd->xof); @@ -382,17 +511,18 @@ static void view_pan_exit(bContext *C, wmOperator *op, bool cancel) static int view_pan_exec(bContext *C, wmOperator *op) { SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); float offset[2]; RNA_float_get_array(op->ptr, "offset", offset); if (sc->flag & SC_LOCK_SELECTION) { - sc->xlockof += offset[0]; - sc->ylockof += offset[1]; + rsc->xlockof += offset[0]; + rsc->ylockof += offset[1]; } else { - sc->xof += offset[0]; - sc->yof += offset[1]; + rsc->xof += offset[0]; + rsc->yof += offset[1]; } ED_region_tag_redraw(CTX_wm_region(C)); @@ -403,11 +533,11 @@ static int view_pan_exec(bContext *C, wmOperator *op) static int view_pan_invoke(bContext *C, wmOperator *op, const wmEvent *event) { if (event->type == MOUSEPAN) { - SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); float offset[2]; - offset[0] = (event->prevx - event->x) / sc->zoom; - offset[1] = (event->prevy - event->y) / sc->zoom; + offset[0] = (event->prevx - event->x) / rsc->zoom; + offset[1] = (event->prevy - event->y) / rsc->zoom; RNA_float_set_array(op->ptr, "offset", offset); @@ -424,15 +554,15 @@ static int view_pan_invoke(bContext *C, wmOperator *op, const wmEvent *event) static int view_pan_modal(bContext *C, wmOperator *op, const wmEvent *event) { - SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); ViewPanData *vpd = op->customdata; float offset[2]; switch (event->type) { case MOUSEMOVE: copy_v2_v2(vpd->vec, &vpd->xorig); - offset[0] = (vpd->x - event->x) / sc->zoom; - offset[1] = (vpd->y - event->y) / sc->zoom; + offset[0] = (vpd->x - event->x) / rsc->zoom; + offset[1] = (vpd->y - event->y) / rsc->zoom; RNA_float_set_array(op->ptr, "offset", offset); view_pan_exec(C, op); break; @@ -497,6 +627,7 @@ typedef struct ViewZoomData { static void view_zoom_init(bContext *C, wmOperator *op, const wmEvent *event) { SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); ARegion *ar = CTX_wm_region(C); ViewZoomData *vpd; @@ -511,7 +642,7 @@ static void view_zoom_init(bContext *C, wmOperator *op, const wmEvent *event) vpd->x = event->x; vpd->y = event->y; - vpd->zoom = sc->zoom; + vpd->zoom = rsc->zoom; vpd->event_type = event->type; ED_clip_mouse_pos(sc, ar, event->mval, vpd->location); @@ -521,11 +652,11 @@ static void view_zoom_init(bContext *C, wmOperator *op, const wmEvent *event) static void view_zoom_exit(bContext *C, wmOperator *op, bool cancel) { - SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); ViewZoomData *vpd = op->customdata; if (cancel) { - sc->zoom = vpd->zoom; + rsc->zoom = vpd->zoom; ED_region_tag_redraw(CTX_wm_region(C)); } @@ -578,7 +709,7 @@ static void view_zoom_apply(bContext *C, float factor; if (U.viewzoom == USER_ZOOM_CONT) { - SpaceClip *sclip = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); double time = PIL_check_seconds_timer(); float time_step = (float)(time - vpd->timer_lastdraw); float fac; @@ -597,7 +728,7 @@ static void view_zoom_apply(bContext *C, zfac = 1.0f + ((fac / 20.0f) * time_step); vpd->timer_lastdraw = time; - factor = (sclip->zoom * zfac) / vpd->zoom; + factor = (rsc->zoom * zfac) / vpd->zoom; } else { float delta = event->x - vpd->x + event->y - vpd->y; @@ -766,13 +897,13 @@ void CLIP_OT_view_zoom_out(wmOperatorType *ot) static int view_zoom_ratio_exec(bContext *C, wmOperator *op) { - SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); sclip_zoom_set(C, RNA_float_get(op->ptr, "ratio"), NULL); /* ensure pixel exact locations for draw */ - sc->xof = (int) sc->xof; - sc->yof = (int) sc->yof; + rsc->xof = (int) rsc->xof; + rsc->yof = (int) rsc->yof; ED_region_tag_redraw(CTX_wm_region(C)); @@ -808,6 +939,7 @@ static int view_all_exec(bContext *C, wmOperator *op) /* retrieve state */ sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); ar = CTX_wm_region(C); ED_space_clip_get_size(sc, &w, &h); @@ -840,7 +972,7 @@ static int view_all_exec(bContext *C, wmOperator *op) sclip_zoom_set(C, 1.0f, NULL); } - sc->xof = sc->yof = 0.0f; + rsc->xof = rsc->yof = 0.0f; ED_region_tag_redraw(ar); @@ -869,11 +1001,11 @@ void CLIP_OT_view_all(wmOperatorType *ot) static int view_selected_exec(bContext *C, wmOperator *UNUSED(op)) { - SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); ARegion *ar = CTX_wm_region(C); - sc->xlockof = 0.0f; - sc->ylockof = 0.0f; + rsc->xlockof = 0.0f; + rsc->ylockof = 0.0f; ED_clip_view_selection(C, ar, 1); ED_region_tag_redraw(ar); @@ -916,6 +1048,10 @@ static void change_frame_apply(bContext *C, wmOperator *op) /* do updates */ BKE_sound_seek_scene(CTX_data_main(C), scene); WM_event_add_notifier(C, NC_SCENE | ND_FRAME, scene); + + SpaceClip *sc = CTX_wm_space_clip(C); + BKE_movieclip_user_set_frame(&sc->user, RNA_int_get(op->ptr, "frame")); + WM_event_add_notifier(C, NC_MOVIECLIP | ND_DISPLAY, NULL); } static int change_frame_exec(bContext *C, wmOperator *op) @@ -1419,7 +1555,7 @@ static int clip_view_ndof_invoke(bContext *C, wmOperator *UNUSED(op), const wmEv if (event->type != NDOF_MOTION) return OPERATOR_CANCELLED; else { - SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); ARegion *ar = CTX_wm_region(C); float pan_vec[3]; @@ -1428,12 +1564,12 @@ static int clip_view_ndof_invoke(bContext *C, wmOperator *UNUSED(op), const wmEv WM_event_ndof_pan_get(ndof, pan_vec, true); - mul_v2_fl(pan_vec, (speed * ndof->dt) / sc->zoom); + mul_v2_fl(pan_vec, (speed * ndof->dt) / rsc->zoom); pan_vec[2] *= -ndof->dt; sclip_zoom_set_factor(C, 1.0f + pan_vec[2], NULL); - sc->xof += pan_vec[0]; - sc->yof += pan_vec[1]; + rsc->xof += pan_vec[0]; + rsc->yof += pan_vec[1]; ED_region_tag_redraw(ar); diff --git a/source/blender/editors/space_clip/clip_utils.c b/source/blender/editors/space_clip/clip_utils.c index e901b9f8026..646541c7bd9 100644 --- a/source/blender/editors/space_clip/clip_utils.c +++ b/source/blender/editors/space_clip/clip_utils.c @@ -224,7 +224,7 @@ void clip_delete_marker(bContext *C, MovieClip *clip, MovieTrackingTrack *track, } } -void clip_view_center_to_point(SpaceClip *sc, float x, float y) +void clip_view_center_to_point(SpaceClip *sc, RegionSpaceClip *rsc, float x, float y) { int width, height; float aspx, aspy; @@ -232,8 +232,8 @@ void clip_view_center_to_point(SpaceClip *sc, float x, float y) ED_space_clip_get_size(sc, &width, &height); ED_space_clip_get_aspect(sc, &aspx, &aspy); - sc->xof = (x - 0.5f) * width * aspx; - sc->yof = (y - 0.5f) * height * aspy; + rsc->xof = (x - 0.5f) * width * aspx; + rsc->yof = (y - 0.5f) * height * aspy; } void clip_draw_cfra(SpaceClip *sc, ARegion *ar, Scene *scene) diff --git a/source/blender/editors/space_clip/space_clip.c b/source/blender/editors/space_clip/space_clip.c index 05e69968e35..62e036d6e90 100644 --- a/source/blender/editors/space_clip/space_clip.c +++ b/source/blender/editors/space_clip/space_clip.c @@ -236,7 +236,6 @@ static SpaceLink *clip_new(const bContext *C) sc->spacetype = SPACE_CLIP; sc->flag = SC_SHOW_MARKER_PATTERN | SC_SHOW_TRACK_PATH | SC_SHOW_GRAPH_TRACKS_MOTION | SC_SHOW_GRAPH_FRAMES | SC_SHOW_GPENCIL; - sc->zoom = 1.0f; sc->path_length = 20; sc->scopes.track_preview_height = 120; sc->around = V3D_AROUND_LOCAL_ORIGINS; @@ -291,6 +290,12 @@ static SpaceLink *clip_new(const bContext *C) BLI_addtail(&sc->regionbase, ar); ar->regiontype = RGN_TYPE_WINDOW; + /* region data for main region */ + RegionSpaceClip *rsc = MEM_callocN(sizeof(RegionSpaceClip), "region data for clip"); + rsc->zoom = 1.0f; + rsc->flag = RSC_MAIN_CLIP; + ar->regiondata = rsc; + return (SpaceLink *) sc; } @@ -418,6 +423,7 @@ static void clip_operatortypes(void) { /* ** clip_ops.c ** */ WM_operatortype_append(CLIP_OT_open); + WM_operatortype_append(CLIP_OT_open_secondary); WM_operatortype_append(CLIP_OT_reload); WM_operatortype_append(CLIP_OT_view_pan); WM_operatortype_append(CLIP_OT_view_zoom); @@ -519,6 +525,11 @@ static void clip_operatortypes(void) WM_operatortype_append(CLIP_OT_keyframe_insert); WM_operatortype_append(CLIP_OT_keyframe_delete); + /* Correspondence */ + WM_operatortype_append(CLIP_OT_add_correspondence); + WM_operatortype_append(CLIP_OT_delete_correspondence); + WM_operatortype_append(CLIP_OT_solve_multiview); + /* ** clip_graph_ops.c ** */ /* graph editing */ @@ -551,6 +562,8 @@ static void clip_keymap(struct wmKeyConfig *keyconf) keymap = WM_keymap_find(keyconf, "Clip", SPACE_CLIP, 0); WM_keymap_add_item(keymap, "CLIP_OT_open", OKEY, KM_PRESS, KM_ALT, 0); + //TODO(tianwei): think about a shortcut for open_secondary clip + //WM_keymap_add_item(keymap, "CLIP_OT_open_secondary", OKEY, KM_PRESS, KM_ALT, 0); WM_keymap_add_item(keymap, "CLIP_OT_tools", TKEY, KM_PRESS, 0, 0); WM_keymap_add_item(keymap, "CLIP_OT_properties", NKEY, KM_PRESS, 0, 0); @@ -876,6 +889,8 @@ static void clip_dropboxes(void) ListBase *lb = WM_dropboxmap_find("Clip", SPACE_CLIP, 0); WM_dropbox_add(lb, "CLIP_OT_open", clip_drop_poll, clip_drop_copy); + //TODO(tianwei): think about dropbox for secondary clip + //WM_dropbox_add(lb, "CLIP_OT_open_secondary", clip_drop_poll, clip_drop_copy); } static void clip_refresh(const bContext *C, ScrArea *sa) @@ -1085,6 +1100,7 @@ static void clip_refresh(const bContext *C, ScrArea *sa) static void movieclip_main_area_set_view2d(const bContext *C, ARegion *ar) { SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = (RegionSpaceClip*) ar->regiondata; float x1, y1, w, h, aspx, aspy; int width, height, winx, winy; @@ -1107,19 +1123,19 @@ static void movieclip_main_area_set_view2d(const bContext *C, ARegion *ar) ar->v2d.mask.ymax = winy; /* which part of the image space do we see? */ - x1 = ar->winrct.xmin + (winx - sc->zoom * w) / 2.0f; - y1 = ar->winrct.ymin + (winy - sc->zoom * h) / 2.0f; + x1 = ar->winrct.xmin + (winx - rsc->zoom * w) / 2.0f; + y1 = ar->winrct.ymin + (winy - rsc->zoom * h) / 2.0f; - x1 -= sc->zoom * sc->xof; - y1 -= sc->zoom * sc->yof; + x1 -= rsc->zoom * rsc->xof; + y1 -= rsc->zoom * rsc->yof; /* relative display right */ - ar->v2d.cur.xmin = (ar->winrct.xmin - (float)x1) / sc->zoom; - ar->v2d.cur.xmax = ar->v2d.cur.xmin + ((float)winx / sc->zoom); + ar->v2d.cur.xmin = (ar->winrct.xmin - (float)x1) / rsc->zoom; + ar->v2d.cur.xmax = ar->v2d.cur.xmin + ((float)winx / rsc->zoom); /* relative display left */ - ar->v2d.cur.ymin = (ar->winrct.ymin - (float)y1) / sc->zoom; - ar->v2d.cur.ymax = ar->v2d.cur.ymin + ((float)winy / sc->zoom); + ar->v2d.cur.ymin = (ar->winrct.ymin - (float)y1) / rsc->zoom; + ar->v2d.cur.ymax = ar->v2d.cur.ymin + ((float)winy / rsc->zoom); /* normalize 0.0..1.0 */ ar->v2d.cur.xmin /= w; @@ -1151,7 +1167,8 @@ static void clip_main_region_draw(const bContext *C, ARegion *ar) { /* draw entirely, view changes should be handled here */ SpaceClip *sc = CTX_wm_space_clip(C); - MovieClip *clip = ED_space_clip_get_clip(sc); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); float aspx, aspy, zoomx, zoomy, x, y; int width, height; bool show_cursor = false; @@ -1165,12 +1182,12 @@ static void clip_main_region_draw(const bContext *C, ARegion *ar) ImBuf *tmpibuf = NULL; if (clip && clip->tracking.stabilization.flag & TRACKING_2D_STABILIZATION) { - tmpibuf = ED_space_clip_get_stable_buffer(sc, NULL, NULL, NULL); + tmpibuf = ED_space_clip_get_stable_buffer(sc, ar, NULL, NULL, NULL); } if (ED_clip_view_selection(C, ar, 0)) { - sc->xof += sc->xlockof; - sc->yof += sc->ylockof; + rsc->xof += rsc->xlockof; + rsc->yof += rsc->ylockof; } if (tmpibuf) @@ -1251,6 +1268,16 @@ static void clip_main_region_listener(bScreen *UNUSED(sc), ScrArea *UNUSED(sa), } } +static void clip_main_region_free(ARegion *ar) +{ + RegionSpaceClip *rsc = ar->regiondata; + + if (rsc) { + MEM_freeN(rsc); + ar->regiondata = NULL; + } +} + /****************** preview region ******************/ static void clip_preview_region_init(wmWindowManager *wm, ARegion *ar) @@ -1555,6 +1582,7 @@ void ED_spacetype_clip(void) art->init = clip_main_region_init; art->draw = clip_main_region_draw; art->listener = clip_main_region_listener; + art->free = clip_main_region_free; art->keymapflag = ED_KEYMAP_FRAMES | ED_KEYMAP_UI | ED_KEYMAP_GPENCIL; BLI_addhead(&st->regiontypes, art); diff --git a/source/blender/editors/space_clip/tracking_ops.c b/source/blender/editors/space_clip/tracking_ops.c index 169eb76399b..8b9f515995a 100644 --- a/source/blender/editors/space_clip/tracking_ops.c +++ b/source/blender/editors/space_clip/tracking_ops.c @@ -93,6 +93,7 @@ static bool add_marker(const bContext *C, float x, float y) static int add_marker_exec(bContext *C, wmOperator *op) { SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); MovieClip *clip = ED_space_clip_get_clip(sc); float pos[2]; @@ -105,8 +106,8 @@ static int add_marker_exec(bContext *C, wmOperator *op) /* Reset offset from locked position, so frame jumping wouldn't be so * confusing. */ - sc->xlockof = 0; - sc->ylockof = 0; + rsc->xlockof = 0; + rsc->ylockof = 0; WM_event_add_notifier(C, NC_MOVIECLIP | NA_EDITED, clip); @@ -594,6 +595,7 @@ MovieTrackingTrack *tracking_marker_check_slide(bContext *C, { const float distance_clip_squared = 12.0f * 12.0f; SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); ARegion *ar = CTX_wm_region(C); MovieClip *clip = ED_space_clip_get_clip(sc); @@ -724,7 +726,7 @@ MovieTrackingTrack *tracking_marker_check_slide(bContext *C, track = track->next; } - if (global_min_distance_squared < distance_clip_squared / sc->zoom) { + if (global_min_distance_squared < distance_clip_squared / rsc->zoom) { if (area_r) { *area_r = min_area; } @@ -856,6 +858,7 @@ static void free_slide_data(SlideMarkerData *data) static int slide_marker_modal(bContext *C, wmOperator *op, const wmEvent *event) { SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); ARegion *ar = CTX_wm_region(C); SlideMarkerData *data = (SlideMarkerData *)op->customdata; @@ -881,13 +884,13 @@ static int slide_marker_modal(bContext *C, wmOperator *op, const wmEvent *event) mdelta[0] = event->mval[0] - data->mval[0]; mdelta[1] = event->mval[1] - data->mval[1]; - dx = mdelta[0] / data->width / sc->zoom; + dx = mdelta[0] / data->width / rsc->zoom; if (data->lock) { dy = -dx / data->height * data->width; } else { - dy = mdelta[1] / data->height / sc->zoom; + dy = mdelta[1] / data->height / rsc->zoom; } if (data->accurate) { diff --git a/source/blender/editors/space_clip/tracking_ops_correspondence.c b/source/blender/editors/space_clip/tracking_ops_correspondence.c new file mode 100644 index 00000000000..4c48632634a --- /dev/null +++ b/source/blender/editors/space_clip/tracking_ops_correspondence.c @@ -0,0 +1,460 @@ +/* + * ***** BEGIN GPL LICENSE BLOCK ***** + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2016 Blender Foundation. + * All rights reserved. + * + * + * Contributor(s): Blender Foundation, + * Tianwei Shen + * + * ***** END GPL LICENSE BLOCK ***** + */ + +/** \file blender/editors/space_clip/tracking_ops_correspondence.c + * \ingroup spclip + */ + +#include "MEM_guardedalloc.h" + +#include "DNA_screen_types.h" +#include "DNA_space_types.h" +#include "DNA_camera_types.h" +#include "DNA_object_types.h" + +#include "BLI_utildefines.h" +#include "BLI_ghash.h" +#include "BLI_math.h" +#include "BLI_blenlib.h" +#include "BLI_string.h" + +#include "BKE_main.h" +#include "BKE_context.h" +#include "BKE_movieclip.h" +#include "BKE_tracking.h" +#include "BKE_depsgraph.h" +#include "BKE_report.h" +#include "BKE_global.h" +#include "BKE_library.h" + +#include "WM_api.h" +#include "WM_types.h" + +#include "ED_screen.h" +#include "ED_clip.h" + +#include "RNA_access.h" +#include "RNA_define.h" + +#include "BLT_translation.h" + +#include "clip_intern.h" +#include "tracking_ops_intern.h" + +/********************** add correspondence operator *********************/ + +/* return the pointer to a single selected track, if more than one is selected, return NULL */ +static MovieTrackingTrack *get_single_track(SpaceClip *sc, ListBase *tracksbase) +{ + int num_selected_tracks = 0; + MovieTrackingTrack *selected_track; + for (MovieTrackingTrack *track = tracksbase->first; track; track = track->next) { + if (TRACK_VIEW_SELECTED(sc, track)) { + selected_track = track; + num_selected_tracks++; + } + } + if (num_selected_tracks == 1) { + return selected_track; + } + return NULL; +} + +static int add_correspondence_exec(bContext *C, wmOperator *op) +{ + SpaceClip *sc = CTX_wm_space_clip(C); + /* get primary clip */ + MovieClip *clip = ED_space_clip_get_clip(sc); + MovieTracking *tracking = &clip->tracking; + ListBase *tracksbase = BKE_tracking_get_active_tracks(tracking); + + /* get one track from each clip and link them */ + MovieTrackingTrack *primary_track = NULL, *witness_track = NULL; + + /* get a single selected tracks in the primary camera */ + primary_track = get_single_track(sc, tracksbase); + + /* get a single selected tracks in the witness camera, only one witness camera is allowed */ + MovieClip *second_clip = ED_space_clip_get_secondary_clip(sc); + MovieTracking *second_tracking = &second_clip->tracking; + ListBase *second_tracksbase = BKE_tracking_get_active_tracks(second_tracking); + witness_track = get_single_track(sc, second_tracksbase); + + if (!primary_track || !witness_track) { + BKE_report(op->reports, RPT_ERROR, "Select exactly one track in each clip"); + return OPERATOR_CANCELLED; + } + + // add these correspondence + char error_msg[256] = "\0"; + if (!BKE_tracking_correspondence_add(&(tracking->correspondences), primary_track, witness_track, + clip, second_clip, error_msg, sizeof(error_msg))) { + if (error_msg[0]) + BKE_report(op->reports, RPT_ERROR, error_msg); + return OPERATOR_CANCELLED; + } + + return OPERATOR_FINISHED; +} + +void CLIP_OT_add_correspondence(wmOperatorType *ot) +{ + /* identifiers */ + ot->name = "Add Correspondence"; + ot->idname = "CLIP_OT_add_correspondence"; + ot->description = "Add correspondence between primary camera and witness camera"; + + /* api callbacks */ + ot->exec = add_correspondence_exec; + ot->poll = ED_space_clip_correspondence_poll; + + /* flags */ + ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO; +} + +/********************** delete correspondence operator *********************/ + +static int delete_correspondence_exec(bContext *C, wmOperator *UNUSED(op)) +{ + SpaceClip *sc = CTX_wm_space_clip(C); + MovieClip *clip = ED_space_clip_get_clip(sc); + MovieTracking *tracking = &clip->tracking; + bool changed = false; + + /* Remove track correspondences from correspondence base */ + MovieTrackingObject *object = BKE_tracking_object_get_active(tracking); + ListBase *correspondence_base = &tracking->correspondences; + for (MovieTrackingCorrespondence *corr = correspondence_base->first; + corr != NULL; + corr = corr->next) + { + MovieTrackingTrack *track; + track = BKE_tracking_track_get_named(tracking, object, corr->self_track_name); + if (TRACK_VIEW_SELECTED(sc, track)) { + BLI_freelinkN(correspondence_base, corr); + changed = true; + } + } + + /* Nothing selected now, unlock view so it can be scrolled nice again. */ + sc->flag &= ~SC_LOCK_SELECTION; + + if (changed) { + WM_event_add_notifier(C, NC_MOVIECLIP | NA_EDITED, clip); + } + + return OPERATOR_FINISHED; +} + +void CLIP_OT_delete_correspondence(wmOperatorType *ot) +{ + /* identifiers */ + ot->name = "Delete Correspondence"; + ot->idname = "CLIP_OT_delete_correspondence"; + ot->description = "Delete selected tracker correspondene between primary and witness camera"; + + /* api callbacks */ + ot->invoke = WM_operator_confirm; + ot->exec = delete_correspondence_exec; + ot->poll = ED_space_clip_correspondence_poll; + + /* flags */ + ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO; +} + +/********************** solve multiview operator *********************/ + +typedef struct { + Scene *scene; + int clip_num; /* the number of active clips for multi-view reconstruction*/ + MovieClip **clips; /* a clip pointer array that records all the clip pointers */ + MovieClipUser user; + ReportList *reports; + char stats_message[256]; + struct MovieMultiviewReconstructContext *context; +} SolveMultiviewJob; + +/* initialize multiview reconstruction solve + * which is assumed to be triggered only in the primary clip + */ +static bool solve_multiview_initjob(bContext *C, + SolveMultiviewJob *smj, + wmOperator *op, + char *error_msg, + int max_error) +{ + SpaceClip *sc = CTX_wm_space_clip(C); + MovieClip *clip = ED_space_clip_get_clip(sc); + Scene *scene = CTX_data_scene(C); + MovieTracking *tracking = &clip->tracking; + MovieTrackingObject *object = BKE_tracking_object_get_active(tracking); + int width, height; + + /* count all clips number, primary clip will always be the first + * iterate over bContext->data.main.movieclip to get open clips. + */ + Main *main = CTX_data_main(C); + ListBase *mc_base = &(main->movieclip); + smj->clip_num = BLI_listbase_count(mc_base); + printf("%d open clips for reconstruction\n", smj->clip_num); + smj->clips = MEM_callocN(smj->clip_num * sizeof(MovieClip*), "multiview clip pointers"); + smj->clips[0] = clip; + + /* do multi-view reconstruction, fill in witness clips from Main.movieclip */ + int mc_counter = 1; + for (Link *link = main->movieclip.first; + link != NULL; + link = link->next) + { + MovieClip *mc_link = (MovieClip*) link; + if (mc_link != smj->clips[0]) { + smj->clips[mc_counter++] = mc_link; + } + } + BLI_assert(mc_counter == smj->clip_num); + + if (!BKE_tracking_multiview_reconstruction_check(smj->clips, + object, + error_msg, + max_error)) + { + return false; + } + + /* Could fail if footage uses images with different sizes. */ + BKE_movieclip_get_size(clip, &sc->user, &width, &height); + + smj->scene = scene; + smj->reports = op->reports; + smj->user = sc->user; + + // create multiview reconstruction context and pass the tracks and markers to libmv + smj->context = BKE_tracking_multiview_reconstruction_context_new(smj->clips, + smj->clip_num, + object, + object->keyframe1, + object->keyframe2, + width, + height); + printf("new multiview reconstruction context\n"); + + tracking->stats = MEM_callocN(sizeof(MovieTrackingStats), "solve multiview stats"); + + return true; +} + +static void solve_multiview_updatejob(void *scv) +{ + SolveMultiviewJob *smj = (SolveMultiviewJob *)scv; + MovieClip *primary_clip = smj->clips[0]; + MovieTracking *tracking = &primary_clip->tracking; + + BLI_strncpy(tracking->stats->message, + smj->stats_message, + sizeof(tracking->stats->message)); +} + +static void solve_multiview_startjob(void *scv, short *stop, short *do_update, float *progress) +{ + SolveMultiviewJob *smj = (SolveMultiviewJob *)scv; + BKE_tracking_multiview_reconstruction_solve(smj->context, + stop, + do_update, + progress, + smj->stats_message, + sizeof(smj->stats_message)); +} + +// TODO(tianwei): not sure about the scene for witness cameras, check with Sergey +static void solve_multiview_freejob(void *scv) +{ + SolveMultiviewJob *smj = (SolveMultiviewJob *)scv; + MovieClip *clip = smj->clips[0]; // primary camera + MovieTracking *tracking = &clip->tracking; + Scene *scene = smj->scene; + int solved; + + if (!smj->context) { + /* job weren't fully initialized due to some error */ + MEM_freeN(smj); + return; + } + + solved = BKE_tracking_multiview_reconstruction_finish(smj->context, smj->clips); + if (!solved) { + BKE_report(smj->reports, + RPT_WARNING, + "Some data failed to reconstruct (see console for details)"); + } + else { + BKE_reportf(smj->reports, + RPT_INFO, + "Average re-projection error: %.3f", + tracking->reconstruction.error); + } + + /* Set the currently solved primary clip as active for scene. */ + if (scene->clip != NULL) { + id_us_min(&clip->id); + } + scene->clip = clip; + id_us_plus(&clip->id); + + /* Set blender camera focal length so result would look fine there. */ + if (scene->camera != NULL && + scene->camera->data && + GS(((ID *) scene->camera->data)->name) == ID_CA) { + Camera *camera = (Camera *)scene->camera->data; + int width, height; + BKE_movieclip_get_size(clip, &smj->user, &width, &height); + BKE_tracking_camera_to_blender(tracking, scene, camera, width, height); + WM_main_add_notifier(NC_OBJECT, camera); + } + + MEM_freeN(tracking->stats); + tracking->stats = NULL; + + DAG_id_tag_update(&clip->id, 0); + + WM_main_add_notifier(NC_MOVIECLIP | NA_EVALUATED, clip); + WM_main_add_notifier(NC_OBJECT | ND_TRANSFORM, NULL); + + /* Update active clip displayed in scene buttons. */ + WM_main_add_notifier(NC_SCENE, scene); + + BKE_tracking_multiview_reconstruction_context_free(smj->context); + MEM_freeN(smj); +} + +static int solve_multiview_exec(bContext *C, wmOperator *op) +{ + SolveMultiviewJob *scj; + char error_msg[256] = "\0"; + scj = MEM_callocN(sizeof(SolveMultiviewJob), "SolveMultiviewJob data"); + if (!solve_multiview_initjob(C, scj, op, error_msg, sizeof(error_msg))) { + if (error_msg[0]) { + BKE_report(op->reports, RPT_ERROR, error_msg); + } + solve_multiview_freejob(scj); + return OPERATOR_CANCELLED; + } + solve_multiview_startjob(scj, NULL, NULL, NULL); + solve_multiview_freejob(scj); + return OPERATOR_FINISHED; +} + +static int solve_multiview_invoke(bContext *C, + wmOperator *op, + const wmEvent *UNUSED(event)) +{ + SolveMultiviewJob *scj; + ScrArea *sa = CTX_wm_area(C); + SpaceClip *sc = CTX_wm_space_clip(C); + MovieClip *clip = ED_space_clip_get_clip(sc); + MovieTracking *tracking = &clip->tracking; + MovieTrackingReconstruction *reconstruction = + BKE_tracking_get_active_reconstruction(tracking); + wmJob *wm_job; + char error_msg[256] = "\0"; + + if (WM_jobs_test(CTX_wm_manager(C), sa, WM_JOB_TYPE_ANY)) { + /* only one solve is allowed at a time */ + return OPERATOR_CANCELLED; + } + + scj = MEM_callocN(sizeof(SolveMultiviewJob), "SolveCameraJob data"); + if (!solve_multiview_initjob(C, scj, op, error_msg, sizeof(error_msg))) { + if (error_msg[0]) { + BKE_report(op->reports, RPT_ERROR, error_msg); + } + solve_multiview_freejob(scj); + return OPERATOR_CANCELLED; + } + + BLI_strncpy(tracking->stats->message, + "Solving multiview | Preparing solve", + sizeof(tracking->stats->message)); + + /* Hide reconstruction statistics from previous solve. */ + reconstruction->flag &= ~TRACKING_RECONSTRUCTED; + WM_event_add_notifier(C, NC_MOVIECLIP | NA_EVALUATED, clip); + + /* Setup job. */ + wm_job = WM_jobs_get(CTX_wm_manager(C), CTX_wm_window(C), sa, "Solve Camera", + WM_JOB_PROGRESS, WM_JOB_TYPE_CLIP_SOLVE_CAMERA); + WM_jobs_customdata_set(wm_job, scj, solve_multiview_freejob); + WM_jobs_timer(wm_job, 0.1, NC_MOVIECLIP | NA_EVALUATED, 0); + WM_jobs_callbacks(wm_job, + solve_multiview_startjob, + NULL, + solve_multiview_updatejob, + NULL); + + G.is_break = false; + + WM_jobs_start(CTX_wm_manager(C), wm_job); + WM_cursor_wait(0); + + /* add modal handler for ESC */ + WM_event_add_modal_handler(C, op); + + return OPERATOR_RUNNING_MODAL; +} + +static int solve_multiview_modal(bContext *C, + wmOperator *UNUSED(op), + const wmEvent *event) +{ + /* No running solver, remove handler and pass through. */ + if (0 == WM_jobs_test(CTX_wm_manager(C), CTX_wm_area(C), WM_JOB_TYPE_ANY)) + return OPERATOR_FINISHED | OPERATOR_PASS_THROUGH; + + /* Running solver. */ + switch (event->type) { + case ESCKEY: + return OPERATOR_RUNNING_MODAL; + } + + return OPERATOR_PASS_THROUGH; +} + +void CLIP_OT_solve_multiview(wmOperatorType *ot) +{ + /* identifiers */ + ot->name = "Solve multi-view reconstruction"; + ot->idname = "CLIP_OT_solve_multiview"; + ot->description = "Solve multiview reconstruction"; + + /* api callbacks */ + ot->exec = solve_multiview_exec; + ot->invoke = solve_multiview_invoke; + ot->modal = solve_multiview_modal; + ot->poll = ED_space_clip_correspondence_poll; + + /* flags */ + ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO; +} diff --git a/source/blender/editors/space_clip/tracking_ops_plane.c b/source/blender/editors/space_clip/tracking_ops_plane.c index 4332f3ea765..f945c6baf9d 100644 --- a/source/blender/editors/space_clip/tracking_ops_plane.c +++ b/source/blender/editors/space_clip/tracking_ops_plane.c @@ -138,6 +138,7 @@ static MovieTrackingPlaneTrack *tracking_plane_marker_check_slide( { const float distance_clip_squared = 12.0f * 12.0f; SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); ARegion *ar = CTX_wm_region(C); MovieClip *clip = ED_space_clip_get_clip(sc); MovieTracking *tracking = &clip->tracking; @@ -180,7 +181,7 @@ static MovieTrackingPlaneTrack *tracking_plane_marker_check_slide( } } - if (min_distance_squared < distance_clip_squared / sc->zoom) { + if (min_distance_squared < distance_clip_squared / rsc->zoom) { if (corner_r != NULL) { *corner_r = min_corner; } @@ -286,6 +287,7 @@ static int slide_plane_marker_modal(bContext *C, const wmEvent *event) { SpaceClip *sc = CTX_wm_space_clip(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); MovieClip *clip = ED_space_clip_get_clip(sc); SlidePlaneMarkerData *data = (SlidePlaneMarkerData *) op->customdata; float dx, dy, mdelta[2]; @@ -307,8 +309,8 @@ static int slide_plane_marker_modal(bContext *C, mdelta[0] = event->mval[0] - data->previous_mval[0]; mdelta[1] = event->mval[1] - data->previous_mval[1]; - dx = mdelta[0] / data->width / sc->zoom; - dy = mdelta[1] / data->height / sc->zoom; + dx = mdelta[0] / data->width / rsc->zoom; + dy = mdelta[1] / data->height / rsc->zoom; if (data->accurate) { dx /= 5.0f; diff --git a/source/blender/editors/space_clip/tracking_select.c b/source/blender/editors/space_clip/tracking_select.c index e970b1b9743..59d17236a08 100644 --- a/source/blender/editors/space_clip/tracking_select.c +++ b/source/blender/editors/space_clip/tracking_select.c @@ -270,7 +270,9 @@ void ed_tracking_delect_all_plane_tracks(ListBase *plane_tracks_base) static int mouse_select(bContext *C, float co[2], int extend) { SpaceClip *sc = CTX_wm_space_clip(C); - MovieClip *clip = ED_space_clip_get_clip(sc); + ARegion *ar = CTX_wm_region(C); + RegionSpaceClip *rsc = CTX_wm_region_clip(C); + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); MovieTracking *tracking = &clip->tracking; ListBase *tracksbase = BKE_tracking_get_active_tracks(tracking); ListBase *plane_tracks_base = BKE_tracking_get_active_plane_tracks(tracking); @@ -339,8 +341,8 @@ static int mouse_select(bContext *C, float co[2], int extend) } if (!extend) { - sc->xlockof = 0.0f; - sc->ylockof = 0.0f; + rsc->xlockof = 0.0f; + rsc->ylockof = 0.0f; } BKE_tracking_dopesheet_tag_update(tracking); @@ -384,7 +386,7 @@ static int select_invoke(bContext *C, wmOperator *op, const wmEvent *event) MovieTrackingTrack *track = tracking_marker_check_slide(C, event, NULL, NULL, NULL); if (track) { - MovieClip *clip = ED_space_clip_get_clip(sc); + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); clip->tracking.act_track = track; @@ -429,7 +431,7 @@ static int border_select_exec(bContext *C, wmOperator *op) SpaceClip *sc = CTX_wm_space_clip(C); ARegion *ar = CTX_wm_region(C); - MovieClip *clip = ED_space_clip_get_clip(sc); + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); MovieTracking *tracking = &clip->tracking; MovieTrackingTrack *track; MovieTrackingPlaneTrack *plane_track; @@ -539,7 +541,7 @@ static int do_lasso_select_marker(bContext *C, const int mcords[][2], const shor SpaceClip *sc = CTX_wm_space_clip(C); ARegion *ar = CTX_wm_region(C); - MovieClip *clip = ED_space_clip_get_clip(sc); + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); MovieTracking *tracking = &clip->tracking; MovieTrackingTrack *track; MovieTrackingPlaneTrack *plane_track; @@ -684,7 +686,7 @@ static int circle_select_exec(bContext *C, wmOperator *op) SpaceClip *sc = CTX_wm_space_clip(C); ARegion *ar = CTX_wm_region(C); - MovieClip *clip = ED_space_clip_get_clip(sc); + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); MovieTracking *tracking = &clip->tracking; MovieTrackingTrack *track; MovieTrackingPlaneTrack *plane_track; @@ -793,7 +795,8 @@ void CLIP_OT_select_circle(wmOperatorType *ot) static int select_all_exec(bContext *C, wmOperator *op) { SpaceClip *sc = CTX_wm_space_clip(C); - MovieClip *clip = ED_space_clip_get_clip(sc); + ARegion *ar = CTX_wm_region(C); + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); MovieTracking *tracking = &clip->tracking; MovieTrackingTrack *track = NULL; /* selected track */ MovieTrackingPlaneTrack *plane_track = NULL; /* selected plane track */ @@ -912,7 +915,8 @@ void CLIP_OT_select_all(wmOperatorType *ot) static int select_groped_exec(bContext *C, wmOperator *op) { SpaceClip *sc = CTX_wm_space_clip(C); - MovieClip *clip = ED_space_clip_get_clip(sc); + ARegion *ar = CTX_wm_region(C); + MovieClip *clip = ED_space_clip_get_clip_in_region(sc, ar); MovieTrackingTrack *track; MovieTrackingMarker *marker; MovieTracking *tracking = &clip->tracking; diff --git a/source/blender/makesdna/DNA_space_types.h b/source/blender/makesdna/DNA_space_types.h index 5e015544dc9..1f5f41476e8 100644 --- a/source/blender/makesdna/DNA_space_types.h +++ b/source/blender/makesdna/DNA_space_types.h @@ -1262,12 +1262,13 @@ typedef struct SpaceClip { ListBase regionbase; /* storage of regions for inactive spaces */ int spacetype; - float xof, yof; /* user defined offset, image is centered */ - float xlockof, ylockof; /* user defined offset from locked position */ - float zoom; /* user defined zoom level */ + float xof DNA_DEPRECATED, yof DNA_DEPRECATED; /* user defined offset, image is centered */ + float xlockof DNA_DEPRECATED, ylockof DNA_DEPRECATED; /* user defined offset from locked position */ + float zoom DNA_DEPRECATED; /* user defined zoom level */ struct MovieClipUser user; /* user of clip */ struct MovieClip *clip; /* clip data */ + struct MovieClip *secondary_clip; /* secondary clip data */ struct MovieClipScopes scopes; /* different scoped displayed in space panels */ int flag; /* flags */ @@ -1295,6 +1296,19 @@ typedef struct SpaceClip { MaskSpaceInfo mask_info; } SpaceClip; +typedef enum eRegionSpaceClip_Flag { + RSC_MAIN_CLIP = (1 << 0), + RSC_SECONDARY_CLIP = (1 << 1), +} eRegionSpaceClip_Flag; + +/* region-related settings for Clip Editor */ +typedef struct RegionSpaceClip { + float xof, yof; /* user defined offset, image is centered */ + float xlockof, ylockof; /* user defined offset from locked position */ + float zoom; /* user defined zoom level */ + int flag; /* region type (main clip/secondary_clip), used in correspondence mode */ +} RegionSpaceClip; + /* SpaceClip->flag */ typedef enum eSpaceClip_Flag { SC_SHOW_MARKER_PATTERN = (1 << 0), @@ -1328,6 +1342,7 @@ typedef enum eSpaceClip_Mode { /*SC_MODE_RECONSTRUCTION = 1,*/ /* DEPRECATED */ /*SC_MODE_DISTORTION = 2,*/ /* DEPRECATED */ SC_MODE_MASKEDIT = 3, + SC_MODE_CORRESPONDENCE = 4, } eSpaceClip_Mode; /* SpaceClip->view */ diff --git a/source/blender/makesdna/DNA_tracking_types.h b/source/blender/makesdna/DNA_tracking_types.h index 42b72c1ff93..3df4546cef7 100644 --- a/source/blender/makesdna/DNA_tracking_types.h +++ b/source/blender/makesdna/DNA_tracking_types.h @@ -44,10 +44,12 @@ struct bGPdata; struct Image; +struct MovieClip; struct MovieReconstructedCamera; struct MovieTrackingCamera; struct MovieTrackingMarker; struct MovieTrackingTrack; +struct MovieTrackingCorrespondence; struct MovieTracking; typedef struct MovieReconstructedCamera { @@ -164,6 +166,18 @@ typedef struct MovieTrackingTrack { float weight_stab; } MovieTrackingTrack; +typedef struct MovieTrackingCorrespondence { + struct MovieTrackingCorrespondence *next, *prev; + + char name[64]; /* MAX_NAME */ + + char self_track_name[64]; + char other_track_name[64]; + + struct MovieClip *self_clip; + struct MovieClip *other_clip; +} MovieTrackingCorrespondence; + typedef struct MovieTrackingPlaneMarker { /* Corners of the plane in the following order: * @@ -348,6 +362,7 @@ typedef struct MovieTracking { MovieTrackingCamera camera; /* camera intrinsics */ ListBase tracks; /* list of tracks used for camera object */ ListBase plane_tracks; /* list of plane tracks used by camera object */ + ListBase correspondences; /* list of correspondence for multi-view support */ MovieTrackingReconstruction reconstruction; /* reconstruction data for camera object */ MovieTrackingStabilization stabilization; /* stabilization data */ MovieTrackingTrack *act_track; /* active track */ diff --git a/source/blender/makesdna/DNA_userdef_types.h b/source/blender/makesdna/DNA_userdef_types.h index 5e7e7366e35..61b580c0904 100644 --- a/source/blender/makesdna/DNA_userdef_types.h +++ b/source/blender/makesdna/DNA_userdef_types.h @@ -298,6 +298,7 @@ typedef struct ThemeSpace { char clipping_border_3d[4]; char marker_outline[4], marker[4], act_marker[4], sel_marker[4], dis_marker[4], lock_marker[4]; + char linked_marker[4], sel_linked_marker[4]; /* cross-clip linked markers and selected linked markers */ char bundle_solid[4]; char path_before[4], path_after[4]; char camera_path[4]; diff --git a/source/blender/makesrna/intern/rna_space.c b/source/blender/makesrna/intern/rna_space.c index 5e364a3adf1..e794d194071 100644 --- a/source/blender/makesrna/intern/rna_space.c +++ b/source/blender/makesrna/intern/rna_space.c @@ -189,6 +189,7 @@ EnumPropertyItem rna_enum_viewport_shade_items[] = { EnumPropertyItem rna_enum_clip_editor_mode_items[] = { {SC_MODE_TRACKING, "TRACKING", ICON_ANIM_DATA, "Tracking", "Show tracking and solving tools"}, {SC_MODE_MASKEDIT, "MASK", ICON_MOD_MASK, "Mask", "Show mask editing tools"}, + {SC_MODE_CORRESPONDENCE, "CORRESPONDENCE", ICON_MOD_CORRESPONDENCE, "Correspondence", "Show correspondence editing tools"}, {0, NULL, 0, NULL, NULL} }; @@ -1558,6 +1559,14 @@ static void rna_SpaceClipEditor_clip_set(PointerRNA *ptr, PointerRNA value) ED_space_clip_set_clip(NULL, screen, sc, (MovieClip *)value.data); } +static void rna_SpaceClipEditor_secondary_clip_set(PointerRNA *ptr, PointerRNA value) +{ + SpaceClip *sc = (SpaceClip *)(ptr->data); + bScreen *screen = (bScreen *)ptr->id.data; + + ED_space_clip_set_secondary_clip(NULL, screen, sc, (MovieClip *)value.data); +} + static void rna_SpaceClipEditor_mask_set(PointerRNA *ptr, PointerRNA value) { SpaceClip *sc = (SpaceClip *)(ptr->data); @@ -1565,19 +1574,19 @@ static void rna_SpaceClipEditor_mask_set(PointerRNA *ptr, PointerRNA value) ED_space_clip_set_mask(NULL, sc, (Mask *)value.data); } -static void rna_SpaceClipEditor_clip_mode_update(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr) +static void rna_SpaceClipEditor_clip_mode_update(bContext *C, PointerRNA *ptr) { SpaceClip *sc = (SpaceClip *)(ptr->data); sc->scopes.ok = 0; + + /* update split view if in correspondence mode */ + ED_clip_update_correspondence_mode(C, sc); } -static void rna_SpaceClipEditor_lock_selection_update(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr) +static void rna_SpaceClipEditor_lock_selection_update(bContext *C, PointerRNA *ptr) { - SpaceClip *sc = (SpaceClip *)(ptr->data); - - sc->xlockof = 0.f; - sc->ylockof = 0.f; + ED_space_clip_region_set_lock_zero(C); } static void rna_SpaceClipEditor_view_type_update(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr) @@ -4548,6 +4557,13 @@ static void rna_def_space_clip(BlenderRNA *brna) RNA_def_property_pointer_funcs(prop, NULL, "rna_SpaceClipEditor_clip_set", NULL, NULL); RNA_def_property_update(prop, NC_SPACE | ND_SPACE_CLIP, NULL); + /* secondary movieclip */ + prop = RNA_def_property(srna, "secondary_clip", PROP_POINTER, PROP_NONE); + RNA_def_property_flag(prop, PROP_EDITABLE); + RNA_def_property_ui_text(prop, "Secondary Movie Clip", "Secondary Movie clip displayed and edited in this space"); + RNA_def_property_pointer_funcs(prop, NULL, "rna_SpaceClipEditor_secondary_clip_set", NULL, NULL); + RNA_def_property_update(prop, NC_SPACE | ND_SPACE_CLIP, NULL); + /* clip user */ prop = RNA_def_property(srna, "clip_user", PROP_POINTER, PROP_NONE); RNA_def_property_flag(prop, PROP_NEVER_NULL); @@ -4565,6 +4581,7 @@ static void rna_def_space_clip(BlenderRNA *brna) RNA_def_property_enum_sdna(prop, NULL, "mode"); RNA_def_property_enum_items(prop, rna_enum_clip_editor_mode_items); RNA_def_property_ui_text(prop, "Mode", "Editing context being displayed"); + RNA_def_property_flag(prop, PROP_CONTEXT_UPDATE); RNA_def_property_update(prop, NC_SPACE | ND_SPACE_CLIP, "rna_SpaceClipEditor_clip_mode_update"); /* view */ @@ -4591,6 +4608,7 @@ static void rna_def_space_clip(BlenderRNA *brna) prop = RNA_def_property(srna, "lock_selection", PROP_BOOLEAN, PROP_NONE); RNA_def_property_ui_text(prop, "Lock to Selection", "Lock viewport to selected markers during playback"); RNA_def_property_boolean_sdna(prop, NULL, "flag", SC_LOCK_SELECTION); + RNA_def_property_flag(prop, PROP_CONTEXT_UPDATE); RNA_def_property_update(prop, NC_SPACE | ND_SPACE_CLIP, "rna_SpaceClipEditor_lock_selection_update"); /* lock to time cursor */ diff --git a/source/blender/makesrna/intern/rna_userdef.c b/source/blender/makesrna/intern/rna_userdef.c index d1e89ea18d0..629a42e92bd 100644 --- a/source/blender/makesrna/intern/rna_userdef.c +++ b/source/blender/makesrna/intern/rna_userdef.c @@ -2899,6 +2899,18 @@ static void rna_def_userdef_theme_space_clip(BlenderRNA *brna) RNA_def_property_ui_text(prop, "Selected Marker", "Color of selected marker"); RNA_def_property_update(prop, 0, "rna_userdef_update"); + prop = RNA_def_property(srna, "linked_marker", PROP_FLOAT, PROP_COLOR_GAMMA); + RNA_def_property_float_sdna(prop, NULL, "linked_marker"); + RNA_def_property_array(prop, 3); + RNA_def_property_ui_text(prop, "Linked Marker", "Color of linked marker"); + RNA_def_property_update(prop, 0, "rna_userdef_update"); + + prop = RNA_def_property(srna, "selected_linked_marker", PROP_FLOAT, PROP_COLOR_GAMMA); + RNA_def_property_float_sdna(prop, NULL, "sel_linked_marker"); + RNA_def_property_array(prop, 3); + RNA_def_property_ui_text(prop, "Selected Linked Marker", "Color of selected linked marker"); + RNA_def_property_update(prop, 0, "rna_userdef_update"); + prop = RNA_def_property(srna, "disabled_marker", PROP_FLOAT, PROP_COLOR_GAMMA); RNA_def_property_float_sdna(prop, NULL, "dis_marker"); RNA_def_property_array(prop, 3); diff --git a/source/blenderplayer/bad_level_call_stubs/stubs.c b/source/blenderplayer/bad_level_call_stubs/stubs.c index 300185b3b16..42fce22e148 100644 --- a/source/blenderplayer/bad_level_call_stubs/stubs.c +++ b/source/blenderplayer/bad_level_call_stubs/stubs.c @@ -396,6 +396,9 @@ void ED_screen_set_scene(struct bContext *C, struct bScreen *screen, struct Scen struct MovieClip *ED_space_clip_get_clip(struct SpaceClip *sc) RET_NULL void ED_space_clip_set_clip(struct bContext *C, struct bScreen *screen, struct SpaceClip *sc, struct MovieClip *clip) RET_NONE void ED_space_clip_set_mask(struct bContext *C, struct SpaceClip *sc, struct Mask *mask) RET_NONE +void ED_space_clip_set_secondary_clip(struct bContext *C, struct bScreen *screen, struct SpaceClip *sc, struct MovieClip *secondary_clip) RET_NONE +void ED_clip_update_correspondence_mode(struct bContext *C, struct SpaceClip *sc) RET_NONE +void ED_space_clip_region_set_lock_zero(struct bContext *C) RET_NONE void ED_space_image_set_mask(struct bContext *C, struct SpaceImage *sima, struct Mask *mask) RET_NONE void ED_area_tag_redraw_regiontype(struct ScrArea *sa, int regiontype) RET_NONE |