solr/solr-ref-guide/tools/BuildNavDataFiles.java - lucene-solr - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 import java.io.*;
 import java.io.FilenameFilter;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;

 import org.json.*;

 import org.asciidoctor.Asciidoctor.Factory;
 import org.asciidoctor.Asciidoctor;
 import org.asciidoctor.ast.DocumentHeader;


 public class BuildNavDataFiles {

   public static void main(String[] args) throws Exception {
     if (args.length != 2) {
       throw new RuntimeException("Wrong # of args: " + args.length);
     }

     final File adocDir = new File(args[0]);
     final String mainPageShortname = args[1];
     if (! adocDir.exists()) {
       throw new RuntimeException("asciidoc directory does not exist: " + adocDir.toString());
     }

     // build up a quick mapping of every known page
     System.out.println("Building up tree of all known pages");
     final Map<String,Page> allPages = new LinkedHashMap<String,Page>();
     Asciidoctor doctor = null;
     try {
       doctor = Factory.create();
       final File[] adocFiles = adocDir.listFiles(ADOC_FILE_NAMES);
       for (File file : adocFiles) {
         Page page = new Page(file, doctor.readDocumentHeader(file));
         if (allPages.containsKey(page.shortname)) {
           throw new RuntimeException("multiple pages with same shortname: " + page.file.toString() + " and " + allPages.get(page.shortname));
         }
         allPages.put(page.shortname, page);
       }
     } finally {
       if (null != doctor) {
         doctor.shutdown();
         doctor = null;
       }
     }

     // build up a hierarchical structure rooted at our mainPage
     final Page mainPage = allPages.get(mainPageShortname);
     if (null == mainPage) {
       throw new RuntimeException("no main-page found with shortname: " + mainPageShortname);
     }
     // NOTE: mainPage claims to be its own parent to prevent anyone decendent from introducing a loop
     mainPage.buildPageTreeRecursive(mainPage, allPages);

     { // validate that there are no orphan pages
       int orphans = 0;
       for (Page p : allPages.values()) {
         if (null == p.getParent()) {
           orphans++;
           System.err.println("ERROR: Orphan page: " + p.file);
         }
       }
       if (0 != orphans) {
         throw new RuntimeException("Found " + orphans + " orphan pages (which are not in the 'page-children' attribute of any other pages)");
       }
     }

     // Loop over all files "in order" to build up next/prev maps for use in building the scrollnav
     final Map<String,Page> nextPage = new HashMap<String,Page>();
     final Map<String,Page> prevPage = new HashMap<String,Page>();
     System.out.println("Looping over pages to build nav data");

     { // the previous page seen during our walk
       AtomicReference<Page> previous = new AtomicReference<Page>();

       mainPage.depthFirstWalk(new Page.RecursiveAction() {
         public boolean act(Page page) {
           if (null != previous.get()) {
             // add previous as our 'prev' page, and ourselves as the 'next' of previous
             prevPage.put(page.shortname, previous.get());
             nextPage.put(previous.get().shortname, page);
           }
           previous.set(page);
           return true;
         }
       });
     }

     // Build up the scrollnav file for jekyll's footer
     File scrollnavFile = new File(new File(adocDir, "_data"), "scrollnav.json");
     if (scrollnavFile.exists()) {
       throw new RuntimeException(scrollnavFile.toString() + " already exists");
     }
     System.out.println("Creating " + scrollnavFile.toString());
     try (Writer w = new OutputStreamWriter(new FileOutputStream(scrollnavFile), "UTF-8")) {
       JSONObject scrollnav = new JSONObject();
       for (Page p : allPages.values()) {
         JSONObject current = new JSONObject();
         Page prev = prevPage.get(p.shortname);
         Page next = nextPage.get(p.shortname);
         if (null != prev) {
           current.put("prev",
                       new JSONObject()
                       .put("url", prev.permalink)
                       .put("title", prev.title));
         }
         if (null != next) {
           current.put("next",
                       new JSONObject()
                       .put("url", next.permalink)
                       .put("title", next.title));
         }
         scrollnav.put(p.shortname, current);
       }
       // HACK: jekyll doesn't like escaped forward slashes in it's JSON?
       w.write(scrollnav.toString(2).replaceAll("\\\\/","/"));
     }

     // Build up the sidebar file for jekyll
     File sidebarFile = new File(new File(adocDir, "_data"), "sidebar.json");
     if (sidebarFile.exists()) {
       throw new RuntimeException(sidebarFile.toString() + " already exists");
     }
     System.out.println("Creating " + sidebarFile.toString());
     try (Writer w = new OutputStreamWriter(new FileOutputStream(sidebarFile), "UTF-8")) {
       // A stack for tracking what we're working on as we recurse
       final Stack<JSONObject> stack = new Stack<JSONObject>();

       mainPage.depthFirstWalk(new Page.RecursiveAction() {
         public boolean act(Page page) {
           final int depth = stack.size();
           if (4 < depth) {
             System.err.println("ERROR: depth==" + depth + " for " + page.permalink);
             System.err.println("sidebar.html template can not support pages this deep");
             System.exit(-1);
           }
           try {
             final JSONObject current = new JSONObject()
               .put("title",page.title)
               .put("url", page.permalink)
               .put("depth", depth)
               .put("kids", new JSONArray());

             if (0 < depth) {
               JSONObject parent = stack.peek();
               ((JSONArray)parent.get("kids")).put(current);
             }

             stack.push(current);
           } catch (JSONException e) {
             throw new RuntimeException(e);
           }
           return true;
         }
         public void postKids(Page page) {
           final JSONObject current = stack.pop();
           if (0 == stack.size()) {
             assert page == mainPage;
             try {
               // HACK: jekyll doesn't like escaped forward slashes in it's JSON?
               w.write(current.toString(2).replaceAll("\\\\/","/"));
             } catch (IOException | JSONException e) {
               throw new RuntimeException(e);
             }
           }
         }
       });
     }

   }

   /** Simple struct for modeling the key metadata for dealing with page navigation */
   public static final class Page {
     public final File file;
     public final String title; // NOTE: has html escape codes in it
     public final String shortname;
     public final String permalink;
     public final List<String> kidShortnames;
     /** NOTE: not populated on construction
      * @see #buildPageTreeRecursive
      */
     private Page parent;
     public Page getParent() {
       return parent;
     }
     /** NOTE: not populated on construction
      * @see #buildPageTreeRecursive
      */
     public final List<Page> kids;
     private final List<Page> mutableKids;
     public Page(File file, DocumentHeader header) {
       if (! file.getName().endsWith(".adoc")) {
         throw new RuntimeException(file + " has does not end in '.adoc' - this code can't be used");
       }

       this.file = file;
       this.title = header.getDocumentTitle().getMain();

       this.shortname = file.getName().replaceAll("\\.adoc$","");
       this.permalink = this.shortname + ".html";

       // TODO: do error checking if attribute metadata we care about is missing
       Map<String,Object> attrs = header.getAttributes();

       // See SOLR-11541: Fail if a user adds new docs with older missleading attributes we don't use/want
       for (String attr : Arrays.asList("page-shortname", "page-permalink")) {
         if (attrs.containsKey(attr)) {
           throw new RuntimeException(file + ": remove the " + attr + " attribute, it's no longer needed, and may confuse readers/editors");
         }
       }

       if (attrs.containsKey("page-children")) {
         String kidsString = ((String) attrs.get("page-children")).trim();
         this.kidShortnames = Collections.<String>unmodifiableList
           (Arrays.asList(kidsString.split(",\\s+")));
         this.mutableKids = new ArrayList<Page>(kidShortnames.size());
       } else {
         this.kidShortnames = Collections.<String>emptyList();
         this.mutableKids = Collections.<Page>emptyList();
       }
       this.kids = Collections.<Page>unmodifiableList(mutableKids);
     }

     /**
      * Recursively sets {@link #getParent} and populates {@link #kids} from {@link #kidShortnames}
      * via the <code>allPages</code> Map
      */
     public void buildPageTreeRecursive(Page parent, Map<String,Page> allPages) {
       if (null != parent) {
         if (null != this.parent) {
           // as long as we also check (later) that every page has a parent, this check (prior to recusion)
           // also ensures we never have any loops
           throw new RuntimeException(file.getName() + " is listed as the child of (at least) 2 pages: '" + parent.shortname + "' and '" + this.parent.shortname + "'");
         }
         this.parent = parent;
       }
       for (String kidShortname : kidShortnames) {
         Page kid = allPages.get(kidShortname);
         if (null == kid) {
           throw new RuntimeException("Unable to locate " + kidShortname + "; child of " + shortname + "("+file.toString());
         }
         mutableKids.add(kid);
         kid.buildPageTreeRecursive(this, allPages);
       }
     }

     /**
      * Do a depth first recursive action on this node and it's {@link #kids}
      * @see RecursiveAction
      */
     public void depthFirstWalk(RecursiveAction action) {
       if (action.act(this)) {
         for (Page kid : kids) {
           kid.depthFirstWalk(action);
         }
         action.postKids(this);
       }
     }

     /** @see #depthFirstWalk */
     public static interface RecursiveAction {
       /** return true if kids should also be visited */
       public boolean act(Page page);
       /**
        * called after recusion to each kid (if any) of specified node,
        * never called if {@link #act} returned false
        */
       public default void postKids(Page page) { /* No-op */ };
     }
   }


   /** Trivial filter for only "*.adoc" files */
   public static final FilenameFilter ADOC_FILE_NAMES = new FilenameFilter() {
     public boolean accept(File dir, String name) {
       return name.endsWith(".adoc");
     }
   };
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	import java.io.*;
	import java.io.FilenameFilter;
	import java.util.*;
	import java.util.concurrent.atomic.AtomicInteger;
	import java.util.concurrent.atomic.AtomicReference;

	import org.json.*;

	import org.asciidoctor.Asciidoctor.Factory;
	import org.asciidoctor.Asciidoctor;
	import org.asciidoctor.ast.DocumentHeader;


	public class BuildNavDataFiles {

	public static void main(String[] args) throws Exception {
	if (args.length != 2) {
	throw new RuntimeException("Wrong # of args: " + args.length);
	}

	final File adocDir = new File(args[0]);
	final String mainPageShortname = args[1];
	if (! adocDir.exists()) {
	throw new RuntimeException("asciidoc directory does not exist: " + adocDir.toString());
	}

	// build up a quick mapping of every known page
	System.out.println("Building up tree of all known pages");
	final Map<String,Page> allPages = new LinkedHashMap<String,Page>();
	Asciidoctor doctor = null;
	try {
	doctor = Factory.create();
	final File[] adocFiles = adocDir.listFiles(ADOC_FILE_NAMES);
	for (File file : adocFiles) {
	Page page = new Page(file, doctor.readDocumentHeader(file));
	if (allPages.containsKey(page.shortname)) {
	throw new RuntimeException("multiple pages with same shortname: " + page.file.toString() + " and " + allPages.get(page.shortname));
	}
	allPages.put(page.shortname, page);
	}
	} finally {
	if (null != doctor) {
	doctor.shutdown();
	doctor = null;
	}
	}

	// build up a hierarchical structure rooted at our mainPage
	final Page mainPage = allPages.get(mainPageShortname);
	if (null == mainPage) {
	throw new RuntimeException("no main-page found with shortname: " + mainPageShortname);
	}
	// NOTE: mainPage claims to be its own parent to prevent anyone decendent from introducing a loop
	mainPage.buildPageTreeRecursive(mainPage, allPages);

	{ // validate that there are no orphan pages
	int orphans = 0;
	for (Page p : allPages.values()) {
	if (null == p.getParent()) {
	orphans++;
	System.err.println("ERROR: Orphan page: " + p.file);
	}
	}
	if (0 != orphans) {
	throw new RuntimeException("Found " + orphans + " orphan pages (which are not in the 'page-children' attribute of any other pages)");
	}
	}

	// Loop over all files "in order" to build up next/prev maps for use in building the scrollnav
	final Map<String,Page> nextPage = new HashMap<String,Page>();
	final Map<String,Page> prevPage = new HashMap<String,Page>();
	System.out.println("Looping over pages to build nav data");

	{ // the previous page seen during our walk
	AtomicReference<Page> previous = new AtomicReference<Page>();

	mainPage.depthFirstWalk(new Page.RecursiveAction() {
	public boolean act(Page page) {
	if (null != previous.get()) {
	// add previous as our 'prev' page, and ourselves as the 'next' of previous
	prevPage.put(page.shortname, previous.get());
	nextPage.put(previous.get().shortname, page);
	}
	previous.set(page);
	return true;
	}
	});
	}

	// Build up the scrollnav file for jekyll's footer
	File scrollnavFile = new File(new File(adocDir, "_data"), "scrollnav.json");
	if (scrollnavFile.exists()) {
	throw new RuntimeException(scrollnavFile.toString() + " already exists");
	}
	System.out.println("Creating " + scrollnavFile.toString());
	try (Writer w = new OutputStreamWriter(new FileOutputStream(scrollnavFile), "UTF-8")) {
	JSONObject scrollnav = new JSONObject();
	for (Page p : allPages.values()) {
	JSONObject current = new JSONObject();
	Page prev = prevPage.get(p.shortname);
	Page next = nextPage.get(p.shortname);
	if (null != prev) {
	current.put("prev",
	new JSONObject()
	.put("url", prev.permalink)
	.put("title", prev.title));
	}
	if (null != next) {
	current.put("next",
	new JSONObject()
	.put("url", next.permalink)
	.put("title", next.title));
	}
	scrollnav.put(p.shortname, current);
	}
	// HACK: jekyll doesn't like escaped forward slashes in it's JSON?
	w.write(scrollnav.toString(2).replaceAll("\\\\/","/"));
	}

	// Build up the sidebar file for jekyll
	File sidebarFile = new File(new File(adocDir, "_data"), "sidebar.json");
	if (sidebarFile.exists()) {
	throw new RuntimeException(sidebarFile.toString() + " already exists");
	}
	System.out.println("Creating " + sidebarFile.toString());
	try (Writer w = new OutputStreamWriter(new FileOutputStream(sidebarFile), "UTF-8")) {
	// A stack for tracking what we're working on as we recurse
	final Stack<JSONObject> stack = new Stack<JSONObject>();

	mainPage.depthFirstWalk(new Page.RecursiveAction() {
	public boolean act(Page page) {
	final int depth = stack.size();
	if (4 < depth) {
	System.err.println("ERROR: depth==" + depth + " for " + page.permalink);
	System.err.println("sidebar.html template can not support pages this deep");
	System.exit(-1);
	}
	try {
	final JSONObject current = new JSONObject()
	.put("title",page.title)
	.put("url", page.permalink)
	.put("depth", depth)
	.put("kids", new JSONArray());

	if (0 < depth) {
	JSONObject parent = stack.peek();
	((JSONArray)parent.get("kids")).put(current);
	}

	stack.push(current);
	} catch (JSONException e) {
	throw new RuntimeException(e);
	}
	return true;
	}
	public void postKids(Page page) {
	final JSONObject current = stack.pop();
	if (0 == stack.size()) {
	assert page == mainPage;
	try {
	// HACK: jekyll doesn't like escaped forward slashes in it's JSON?
	w.write(current.toString(2).replaceAll("\\\\/","/"));
	} catch (IOException \| JSONException e) {
	throw new RuntimeException(e);
	}
	}
	}
	});
	}

	}

	/** Simple struct for modeling the key metadata for dealing with page navigation */
	public static final class Page {
	public final File file;
	public final String title; // NOTE: has html escape codes in it
	public final String shortname;
	public final String permalink;
	public final List<String> kidShortnames;
	/** NOTE: not populated on construction
	* @see #buildPageTreeRecursive
	*/
	private Page parent;
	public Page getParent() {
	return parent;
	}
	/** NOTE: not populated on construction
	* @see #buildPageTreeRecursive
	*/
	public final List<Page> kids;
	private final List<Page> mutableKids;
	public Page(File file, DocumentHeader header) {
	if (! file.getName().endsWith(".adoc")) {
	throw new RuntimeException(file + " has does not end in '.adoc' - this code can't be used");
	}

	this.file = file;
	this.title = header.getDocumentTitle().getMain();

	this.shortname = file.getName().replaceAll("\\.adoc$","");
	this.permalink = this.shortname + ".html";

	// TODO: do error checking if attribute metadata we care about is missing
	Map<String,Object> attrs = header.getAttributes();

	// See SOLR-11541: Fail if a user adds new docs with older missleading attributes we don't use/want
	for (String attr : Arrays.asList("page-shortname", "page-permalink")) {
	if (attrs.containsKey(attr)) {
	throw new RuntimeException(file + ": remove the " + attr + " attribute, it's no longer needed, and may confuse readers/editors");
	}
	}

	if (attrs.containsKey("page-children")) {
	String kidsString = ((String) attrs.get("page-children")).trim();
	this.kidShortnames = Collections.<String>unmodifiableList
	(Arrays.asList(kidsString.split(",\\s+")));
	this.mutableKids = new ArrayList<Page>(kidShortnames.size());
	} else {
	this.kidShortnames = Collections.<String>emptyList();
	this.mutableKids = Collections.<Page>emptyList();
	}
	this.kids = Collections.<Page>unmodifiableList(mutableKids);
	}

	/**
	* Recursively sets {@link #getParent} and populates {@link #kids} from {@link #kidShortnames}
	* via the <code>allPages</code> Map
	*/
	public void buildPageTreeRecursive(Page parent, Map<String,Page> allPages) {
	if (null != parent) {
	if (null != this.parent) {
	// as long as we also check (later) that every page has a parent, this check (prior to recusion)
	// also ensures we never have any loops
	throw new RuntimeException(file.getName() + " is listed as the child of (at least) 2 pages: '" + parent.shortname + "' and '" + this.parent.shortname + "'");
	}
	this.parent = parent;
	}
	for (String kidShortname : kidShortnames) {
	Page kid = allPages.get(kidShortname);
	if (null == kid) {
	throw new RuntimeException("Unable to locate " + kidShortname + "; child of " + shortname + "("+file.toString());
	}
	mutableKids.add(kid);
	kid.buildPageTreeRecursive(this, allPages);
	}
	}

	/**
	* Do a depth first recursive action on this node and it's {@link #kids}
	* @see RecursiveAction
	*/
	public void depthFirstWalk(RecursiveAction action) {
	if (action.act(this)) {
	for (Page kid : kids) {
	kid.depthFirstWalk(action);
	}
	action.postKids(this);
	}
	}

	/** @see #depthFirstWalk */
	public static interface RecursiveAction {
	/** return true if kids should also be visited */
	public boolean act(Page page);
	/**
	* called after recusion to each kid (if any) of specified node,
	* never called if {@link #act} returned false
	*/
	public default void postKids(Page page) { /* No-op */ };
	}
	}


	/** Trivial filter for only ".adoc" files /
	public static final FilenameFilter ADOC_FILE_NAMES = new FilenameFilter() {
	public boolean accept(File dir, String name) {
	return name.endsWith(".adoc");
	}
	};
	}