add local indexing back as another task

5 files changed, 113 insertions, 10 deletions

diff --git a/app/views/layouts/layout.html.haml b/app/views/layouts/layout.html.haml
index 4d16b57e..9b0cde25 100644
--- a/app/views/layouts/layout.html.haml
+++ b/app/views/layouts/layout.html.haml
@@ -47,7 +47,7 @@
                   %ul{:class => (@section == 'documentation') ? 'expanded' : ''} 
                     %li= link_to "Reference", "/docs", {:class => (@subsection == 'reference') ? 'active' : ''}
                     %li= link_to "Book", "/book", {:class => (@subsection == 'book') ? 'active' : ''}
-                    %li= link_to "Videos", "/videos", {:class => (@subsection == 'book') ? 'active' : ''}
+                    %li= link_to "Videos", "/videos", {:class => (@subsection == 'videos') ? 'active' : ''}
                     %li= link_to "External Links", "/doc/ext", {:class => (@subsection == 'external-links') ? 'active' : ''}
                 %li
                   = link_to "Downloads", "/downloads", {:class => (@section == 'downloads') ? 'active' : ''}
diff --git a/app/views/site/index.html.haml b/app/views/site/index.html.haml
index f06f5d6d..a01d66f1 100644
--- a/app/views/site/index.html.haml
+++ b/app/views/site/index.html.haml
@@ -55,7 +55,7 @@
   %ul
     %li= link_to "Google", "https://github.com/google", {:class => 'google'}
     %li= link_to "Facebook", "https://github.com/facebook", {:class => 'facebook'}
-    %li= link_to "Blizzard", "https://github.com/Blizzard", {:class => 'blizzard'}
+    %li= link_to "Microsoft", "https://github.com/WindowsAzure", {:class => 'microsoft'}
     %li= link_to "Twitter", "https://github.com/twitter", {:class => 'twitter'}
     %li= link_to "LinkedIn", "https://github.com/linkedin", {:class => 'linked-in'}
     %li= link_to "Linux", "http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git;a=summary", {:class => 'linux'}
diff --git a/db/schema.rb b/db/schema.rb
index 21472c17..90294ce3 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -32,19 +32,19 @@ ActiveRecord::Schema.define(:version => 20120423150252) do
   add_index "chapters", ["book_id"], :name => "index_chapters_on_book_id"
 
   create_table "doc_files", :force => true do |t|
-    t.string    "name"
-    t.timestamp "created_at", :null => false
-    t.timestamp "updated_at", :null => false
+    t.string   "name"
+    t.datetime "created_at", :null => false
+    t.datetime "updated_at", :null => false
   end
 
   add_index "doc_files", ["name"], :name => "index_doc_files_on_name"
 
   create_table "doc_versions", :force => true do |t|
-    t.integer   "version_id"
-    t.integer   "doc_id"
-    t.integer   "doc_file_id"
-    t.timestamp "created_at",  :null => false
-    t.timestamp "updated_at",  :null => false
+    t.integer  "version_id"
+    t.integer  "doc_id"
+    t.integer  "doc_file_id"
+    t.datetime "created_at",  :null => false
+    t.datetime "updated_at",  :null => false
   end
 
   create_table "docs", :force => true do |t|
diff --git a/lib/tasks/local_index.rake b/lib/tasks/local_index.rake
new file mode 100644
index 00000000..69dfa21e
--- /dev/null
+++ b/lib/tasks/local_index.rake
@@ -0,0 +1,95 @@
+require 'asciidoc'
+
+# fill in the db from a local git clone
+task :local_index => :environment do
+  template_dir = File.join(Rails.root, 'templates')
+  dir = ENV["GIT_REPO"]
+  rerun = false
+  Dir.chdir(dir) do
+    # find all tags
+    tags = `git tag | grep v1`.strip.split("\n")
+    tags = tags.select { |tag| tag =~ /v\d([\.\d])+$/ }  # just get release tags
+
+    # for each tag, get a date and a list of file/shas
+    tags.each do |tag|
+
+      puts tag
+
+      stag = Version.where(:name => tag.gsub('v','')).first
+      next if stag && !rerun
+
+      stag = Version.where(:name => tag.gsub('v','')).first_or_create
+
+      # extract metadata
+      commit_sha = `git rev-parse #{tag}`.chomp
+      tree_sha = `git rev-parse #{tag}^{tree}`.chomp
+      tagger = `git cat-file commit #{tag} | grep committer`.chomp.split(' ')
+      tz = tagger.pop
+      ts = tagger.pop
+      ts = Time.at(ts.to_i)
+
+      # save metadata
+      puts "#{tag}: #{ts}, #{commit_sha[0, 8]}, #{tree_sha[0, 8]}"
+      stag.commit_sha = commit_sha
+      stag.tree_sha = tree_sha
+      stag.committed = ts
+      stag.save
+
+      # find all the doc entries
+      entries = `git ls-tree #{tag}:Documentation`.strip.split("\n")
+      tree = entries.map do |e|
+        mode, type, sha, path = e.split(' ')
+        [path, sha, type]
+      end
+      tree = tree.select { |t| t.first =~ /^(git.*|everyday|howto-index,user-manual)\.txt/ }
+
+      puts "Found #{tree.size} entries"
+
+      # generate this tag's command list for includes
+      cmd_list = `git cat-file blob #{tag}:command-list.txt`.split("\n").reject{|l| l =~ /^#/}.inject({}) do |list, cmd|
+        name, kind, attr = cmd.split(/\s+/)
+        list[kind] ||= []
+        list[kind] << [name, attr]
+        list
+      end
+      categories = cmd_list.keys.inject({}) do |list, category|
+        links = cmd_list[category].map do |cmd, attr|
+          if match = `git cat-file blob #{tag}:Documentation/#{cmd}.txt`.match(/NAME\n----\n\S+ - (.*)$/)
+            "linkgit:#{cmd}[1]::\n\t#{attr == 'deprecated' ? '(deprecated) ' : ''}#{match[1]}\n"
+          end
+        end
+
+        list.merge!("cmds-#{category}.txt" => links.compact.join("\n"))
+      end
+
+      tree.each do |entry|
+        path, sha, type = entry
+        path = path.gsub('.txt', '')
+        file = DocFile.where(:name => path).first_or_create
+        doc = Doc.where(:blob_sha => sha).first_or_create
+        if !doc.plain || !doc.html
+          content = `git cat-file blob #{sha}`.chomp
+          asciidoc = Asciidoc::Document.new(path, content) do |inc|
+            if categories.has_key?(inc)
+              categories[inc]
+            else
+              if match = inc.match(/^\.\.\/(.*)$/)
+                git_path = match[1]
+              else
+                git_path = "Documentation/#{inc}"
+              end
+
+              `git cat-file blob #{tag}:#{git_path}`
+            end
+          end
+          doc.plain = asciidoc.source
+          doc.html  = asciidoc.render(template_dir)
+          doc.save
+        end
+        DocVersion.where(:version_id => stag.id, :doc_id => doc.id, :doc_file_id => file.id).first_or_create
+      end
+
+    end
+  end
+end
+
diff --git a/lib/tasks/search.rake b/lib/tasks/search.rake
index 59cc397e..26bd1e01 100644
--- a/lib/tasks/search.rake
+++ b/lib/tasks/search.rake
@@ -6,3 +6,11 @@ task :search_index => :environment do
     p docv.index
   end
 end
+
+require 'pp'
+task :search_index_book => :environment do
+  book = Book.where(:code => 'en').first
+  book.sections.each do |sec|
+    sec.index
+  end
+end