Google Git
Sign in
apache / universal-wiki-converter / 3da293fed8f632d6cb4180455f299631bd8d8b58 / . / src / com / atlassian / uwc / ui / ConverterEngine_v2.java
blob: 7d18e17c03b1e715fbcaa99fcb154e16eb346822 [file] [log] [blame]
package com.atlassian.uwc.ui;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.activation.MimetypesFileTypeMap;
import javax.swing.JFrame;
import javax.swing.JOptionPane;
import javax.swing.ProgressMonitor;
import javax.swing.SwingUtilities;
import org.apache.log4j.Logger;
import org.apache.xmlrpc.XmlRpcException;
import com.atlassian.uwc.converters.Converter;
import com.atlassian.uwc.converters.IllegalLinkNameConverter;
import com.atlassian.uwc.converters.IllegalPageNameConverter;
import com.atlassian.uwc.converters.JavaRegexConverter;
import com.atlassian.uwc.converters.PerlConverter;
import com.atlassian.uwc.converters.twiki.JavaRegexAndTokenizerConverter;
import com.atlassian.uwc.converters.twiki.TWikiRegexConverterCleanerWrapper;
import com.atlassian.uwc.hierarchies.HierarchyBuilder;
import com.atlassian.uwc.hierarchies.HierarchyNode;
import com.atlassian.uwc.ui.xmlrpcwrapperOld.AttachmentForXmlRpcOld;
import com.atlassian.uwc.ui.xmlrpcwrapperOld.PageForXmlRpcOld;
import com.atlassian.uwc.ui.xmlrpcwrapperOld.RemoteWikiBrokerOld;
/**
* This class drives the conversion process by gathering all the files, gathering
* the selected converters, then applying all the converters against each file, then
* sending the converted 'Pages' and attachments to Confluence via XML-RPC (or
* possibly some other method in the future)
*/
public class ConverterEngine_v2 {
private static final String NONCONVERTERTYPE_PAGEHISTORYPRESERVATION = "page-history-preservation";
private static final String NONCONVERTERTYPE_HIERARCHYBUILDER = ".hierarchy-builder";
private static final String CONVERTERTYPE_TWIKICLEANER = ".twiki-cleaner";
private static final String CONVERTERTYPE_JAVAREGEX = ".java-regex";
private static final String CONVERTERTYPE_JAVAREGEXTOKEN = ".java-regex-tokenize";
private static final String CONVERTERTYPE_PERL = ".perl";
private static final String CONVERTERTYPE_CLASS = ".class";
private static final String PROP_ATTACHMENT_SIZE_MAX = "attachment.size.max";
private static String PROP_LOCATION = ConfluenceSettingsForm.CONFLUENCE_SETTINGS_FILE_LOC;
private HashSet<String> attachedFiles;
private String errorMessage;
Logger log = Logger.getLogger("ConverterEngine");
// this logger is used to write out totals for the UWC to a seperate file uwc-totals.log
Logger totalsFileLog = Logger.getLogger("totalsFileLog");
/**
* The string that directory separators (e.g., / on Unix and \ on Windows) are replaced
* with in page titles.
* This is used by DokuWikiLinkConverter too.
*/
public static final String CONFLUENCE_SEPARATOR = " -- ";
protected enum HierarchyHandler {
DEFAULT, //no hierarchy handling
HIERARCHY_BUILDER, //hierarchyBuilder handles
PAGENAME_HIERARCHIES//hierarchy maintained in pagename
};
private HierarchyHandler hierarchyHandler = HierarchyHandler.DEFAULT;
/**
* The mapping from file name extension to mime type that is used when sending
* attachments to Confluence.
*/
private MimetypesFileTypeMap mimeTypes;
/**
* This is the location of the mime type mapping file. For details on the file format,
* refer to the link below.
*
* @see javax.activation.MimetypesFileTypeMap
*/
public final static String mimetypeFileLoc = "conf" + File.separator + "mime.types";
/**
* This field is set if a hierarchy builder "converter" is used. The field controls the
* way in which pages are added/updated in Confluence. If hierarchyBuilder is <code>null</code>, all
* pages are added as top-level pages in the selected space. Otherwise, the hierarchy builder is
* called on to create a page hierarchy, and the engine will insert the pages correspondingly.
*/
private HierarchyBuilder hierarchyBuilder = null;
/**
* This default constructor initializes the mime types.
*/
public ConverterEngine_v2() {
try {
mimeTypes = new MimetypesFileTypeMap(new FileInputStream(mimetypeFileLoc));
} catch (FileNotFoundException e) {
addError("Couldn't load mime types!", e);
}
totalsFileLog.setAdditivity(false);
}
/**
* @param uwcForm
* @todo this method might end up not being needed.....refactor it out?
*/
public void processPages(UWCForm2 uwcForm) {
convert(uwcForm.pageList, uwcForm.engineSelectedConverterList);
// if single page then pop up in editor pane window
// write list to Confluence
}
/**
* High level method to drive the conversion of the 'input pages' via
* the selected 'converterStrings'
*
* @param inputPages - the full path Strings of the files
* @param converterStrings - the full Strings for the converterStrings which have been selected
* for the engine.
* @todo - this isn't currently memory efficient. We should probably switch to streaming
* if people start running out of memory
*/
public void convert(List<File> inputPages, List<String> converterStrings) {
ArrayList<Converter> converters = createConverters(converterStrings);
ConfluenceSettingsForm confSettings = UWCForm2.getInstance().getConfluenceSettingsForm();
// Recurse through directories, adding all files
List<Page> allPages = createPages(confSettings, inputPages);
// Convert the file contents
if (convertPages(allPages, converters)) {
// do final required conversions. This step is seperate, due to state saving issues
boolean useUI = true; //this is necessary, so we can turn it off for unit tests
convertWithRequiredConverters(allPages, useUI);
// write out dialog with metrics of total time and memory ('don't show this again'
// Save the converted pages to disk
// TODO: This is not needed for the converter, and there should really be an option to turn it off.
savePages(allPages, confSettings);
// Finally, send the pages to Confluence if the user approves.
if (okToSend()) {
if (hierarchyHandler == HierarchyHandler.HIERARCHY_BUILDER && hierarchyBuilder != null) {
HierarchyNode root = hierarchyBuilder.buildHierarchy(allPages);
writeHierarchy(root);
} else {
writePages(allPages);
}
}
}
}
/**
* Instantiate all the converterStrings
*
* @param converterStrings a list of converter strings of the form "key=value"
* @return a list of converters
*/
protected ArrayList<Converter> createConverters(List<String> converterStrings) {
ArrayList<Converter> converters = new ArrayList<Converter>();
for (String converterStr : converterStrings) {
Converter converter;
if (isNonConverterProperty(converterStr)) {
handleNonConverterProperty(converterStr);
continue;
}
converter = getConverterFromString(converterStr);
if (converter == null) continue;
converters.add(converter);
}
return converters;
}
/**
* converts the list of pages with the given converter
* @param pages list of pages to be converted
* @param useUI set this to false if you do not want the associated GUI elements
* to be updated. This is useful for unit testing.
*/
protected void convertWithRequiredConverters(List<Page> pages, boolean useUI) {
//create pagename converter and convert with it
String pagenameConvStr = "MyWiki.9999.illegal-names.class=com.atlassian.uwc.converters.IllegalPageNameConverter";
ArrayList<Converter> converters = createOneConverter(pagenameConvStr);
convertPages(pages, converters, useUI, "Checking for illegal pagenames.");
//get the state hashtable
IllegalPageNameConverter pagenameConverter = (IllegalPageNameConverter) converters.remove(0);
HashSet<String> illegalNames = pagenameConverter.getIllegalPagenames();
//create linkname converter and convert with it
String illegallinksConvStr = "MyWiki.9999.illegal-links.class=com.atlassian.uwc.converters.IllegalLinkNameConverter";
converters = createOneConverter(illegallinksConvStr);
IllegalLinkNameConverter linknameConverter = (IllegalLinkNameConverter) converters.get(0);
linknameConverter.setIllegalPagenames(illegalNames);
convertPages(pages, converters, useUI, "Checking for links to illegal pagenames.");
}
/**
* creates the arraylist of converters when only one converter is needed
* @param converterString string representing the converter. Should be in property format. Example:<br/>
* key=value
* @return arraylist with one converter as its sole item
*/
protected ArrayList<Converter> createOneConverter(String converterString) {
ArrayList<String> converterStrings = new ArrayList<String>();
converterStrings.add(converterString);
ArrayList<Converter> converters = createConverters(converterStrings);
return converters;
}
/**
* Instantiates a converter from a correctly formatted String.
* <p/>
* Note: This method is now only called once per converter -- first all converters
* are created, then all pages, then all converters are run on all pages.
*
* @param converterStr A string of the form "name.keyword=parameters". The
* keyword is used to create the correct type of converter, and the parameters
* are then passed to the converter. Finally, the "name.keyword" part is set as
* the key in the converter, mainly for debugging purposes.
* @return converter or null if no converter can be parsed/instantiated
*/
public Converter getConverterFromString(String converterStr) {
Converter converter;
int equalLoc = converterStr.indexOf("=");
String key = converterStr.substring(0, equalLoc);
String value = converterStr.substring(equalLoc + 1);
try {
if (key.indexOf(CONVERTERTYPE_CLASS) >= 0) {
converter = getConverterClassFromCache(value);
} else if (key.indexOf(CONVERTERTYPE_PERL) >= 0) {
converter = PerlConverter.getPerlConverter(value);
converter.setValue(value);
} else if (key.indexOf(CONVERTERTYPE_JAVAREGEXTOKEN) >= 0) {
converter = JavaRegexAndTokenizerConverter.getConverter(value);
converter.setValue(value);
} else if (key.indexOf(CONVERTERTYPE_JAVAREGEX) >= 0) {
converter = JavaRegexConverter.getConverter(value);
converter.setValue(value);
} else if (key.indexOf(CONVERTERTYPE_TWIKICLEANER) >= 0) {
//converter = getConverterClassFromCache(value);
converter = TWikiRegexConverterCleanerWrapper.getTWikiRegexConverterCleanerWrapper(value);
converter.setValue(value);
} else {
addError("Converter ignored -- name pattern not recognized: " + key);
return null;
}
} catch (ClassNotFoundException e) {
addError("Converter ignored -- the Java class " + value + " was not found");
return null;
} catch (IllegalAccessException e) {
addError("Converter ignored -- there was a problem creating a converter object");
return null;
} catch (InstantiationException e) {
addError("Converter ignored -- there was a problem creating the Java class " + value);
return null;
} catch (ClassCastException e) {
addError("Converter ignored -- the Java class " + value +
" must implement the " + Converter.class.getName() + " interface!");
return null;
}
converter.setKey(key);
return converter;
}
/**
* handles necessary state changes for expected properties
* that were set in the converter properties file.
* expected nonconverter properties include hierarchy builder properties
* and page history preservation properties
* @param converterStr should be a line from the converter properties file
* Example:
* MyWiki.0001.someproperty.somepropertytype=setting
* <br/>
* where somepropertytype is an expected property type:
* <br/>
* NONCONVERTERTYPE_HIERARCHYBUILDER or NONCONVERTERTYPE_PAGEHISTORYPRESERVATION
*/
protected void handleNonConverterProperty(String converterStr) {
int equalLoc = converterStr.indexOf("=");
String key = converterStr.substring(0, equalLoc);
String value = converterStr.substring(equalLoc + 1);
try {
if (key.indexOf(NONCONVERTERTYPE_HIERARCHYBUILDER) >= 0) {
if (isHierarchySwitch(key))
setHierarchyHandler(value);
else {
Class c;
c = Class.forName(value);
HierarchyBuilder hierarchy = (HierarchyBuilder) c.newInstance();
hierarchyBuilder = hierarchy;
}
}
else if (key.endsWith(NONCONVERTERTYPE_PAGEHISTORYPRESERVATION)) {
handlePageHistoryProperty(key, value);
}
} catch (ClassNotFoundException e) {
addError("Property ignored -- the Java class " + value + " was not found");
} catch (IllegalAccessException e) {
addError("Property ignored -- there was a problem creating the object: " + value);
} catch (InstantiationException e) {
addError("Property ignored -- there was a problem creating the Java class " + value);
} catch (ClassCastException e) {
addError("Property ignored -- the Java class " + value +
" must implement the " + Converter.class.getName() + " interface!");
}
}
HashMap<String, Converter> converterCacheMap = new HashMap<String, Converter>();
/**
* at long last making some performance enhancements
* here we are creating an object cache which should help a bit
*
* @param key A string representing the converter (actually the part after the
* equals sign of the converter string).
* @return
* @throws ClassNotFoundException
* @throws IllegalAccessException
* @throws InstantiationException
*/
private Converter getConverterClassFromCache(String key) throws ClassNotFoundException, IllegalAccessException, InstantiationException {
Converter converter = (Converter) converterCacheMap.get(key);
if (converter == null) {
Class c = Class.forName(key);
converter = (Converter) c.newInstance();
converterCacheMap.put(key, converter);
}
return converter;
}
/**
* Creates PageForXmlRpcOld objects for all the files in inputPages.
* If any of the files is a directory, it is scanned recursively for files
* matching the pattern in the settings object.
*
* @param confSettings The settings that control the engine.
* @param inputPages A list of files and directories that Pages should be created for.
* @return A list of PageForXmlRpcOld objects for all files matching the pattern in the settings.
*/
protected List<Page> createPages(ConfluenceSettingsForm confSettings, List<File> inputPages) {
List<Page> allPages = new LinkedList<Page>();
final String pattern = confSettings.getPattern();
// This file filter accepts directories and file names ending with the pattern.
// If the pattern is empty, all files are accepted.
FileFilter filter = new FileFilter() {
public boolean accept(File file) {
assert file != null;
return pattern == null ||
"".equals(pattern) ||
file.isDirectory() ||
file.getName().endsWith(pattern);
}
};
for (File fileOrDir : inputPages) {
List<Page> pages = recurse(fileOrDir, filter);
setupPages(fileOrDir, pages);
allPages.addAll(pages);
}
if (isHandlingPageHistories()) {
return sortByHistory(allPages);
}
return allPages;
}
/**
* Recurses through a directory structure and adds all files in it matching the filter.
* Called by createPages.
*
* @param fileOrDir A directory or file. Must not be <code>null</code>.
* @param filter the filter to use when selecting files
* @return A list with PageForXmlRpcOld objects for all the matching files in the directory and its subdirectories
*/
private List<Page> recurse(File fileOrDir, FileFilter filter) {
assert fileOrDir != null;
List<Page> result = new LinkedList<Page>();
if (fileOrDir.isFile() && filter.accept(fileOrDir)) {
result.add(new Page(fileOrDir));
} else if (fileOrDir.isDirectory()) {
File[] files = fileOrDir.listFiles(filter);
for (File file : files) {
result.addAll(recurse(file, filter));
}
}
return result;
}
/**
* Set the names of the pages and performs any other setup needed. Called by recurse().
* If the user selected a directory and this file is inside it, the base directory's
* path is removed and the rest is used as the page name.
* <p/>
* Any directory separators are replaced with the constant CONFLUENCE_SEPARATOR.
*
* @param baseDir The directory that the top-level documents are in
* @param pages A list of pages to set up
*/
protected void setupPages(File baseDir, List<Page> pages) {
String basepath = baseDir.getParentFile().getPath() + File.separator;
int baselength = basepath.length();
for (Page page : pages) {
String pagePath = page.getFile().getPath();
pagePath = pagePath.substring(baselength);
log.debug("pagePath: '" + pagePath + "'");
String pageName = getPagename(pagePath);
//Strip the file name from the path.
int fileNameStart = pagePath.lastIndexOf(File.separator);
if (fileNameStart >= 0) {
pagePath = pagePath.substring(0, fileNameStart);
} else {
pagePath = "";
}
page.setPath(pagePath);
page.setName(pageName);
if (isHandlingPageHistories()) preserveHistory(page, pageName);
log.debug("setupPages() Path: '" + page.getPath() + "', Name: " + page.getName());
}
}
/**
* uses the filename to set the version and name of the given page
* so that the history is preserved in the conversion. Note:
* uses the pageHistorySuffix which is set by the handlePageHistoryProperty
* method
* @param page object that will be changed to reflect pagename and version of given filename
* @param filename should use the pageHistorySuffix to indicate version and pagename:
* <br/>
* if pageHistorySuffix is -#.txt
* <br/>
* then filename should be something like: pagename-2.txt
* @return Page with changed name and version
* Will return passed page with no changes if:
* <ul>
* <li>suffix is null</li>
* <li> suffix has no numerical indicator (#)</li>
* </ul>
*/
protected Page preserveHistory(Page page, String filename) {
//get suffix
String suffix = getPageHistorySuffix();
if (suffix == null) {
log.error("Error attempting to preserve history: Page history suffix is Null.");
return page;
}
//create regex for filename based on the suffix
Matcher hashFinder = hashPattern.matcher(suffix);
String suffixReplaceRegex = "";
if (hashFinder.find()) {
suffixReplaceRegex = hashFinder.replaceAll("(\\\\d+)");
suffixReplaceRegex = "(.*)" + suffixReplaceRegex;
log.debug("new regex: " + suffixReplaceRegex);
}
else {
log.error("Error attempting to preserve history: Suffix is invalid. Must contain '#'.");
return page;
}
//get the version and name
Pattern suffixReplacePattern = Pattern.compile(suffixReplaceRegex);
Matcher suffixReplacer = suffixReplacePattern.matcher(filename);
if (suffixReplacer.find()) {
String pagename = suffixReplacer.group(1);
String versionString = suffixReplacer.group(2);
int version = Integer.parseInt(versionString);
log.debug("version = " + version);
page.setVersion(version);
log.debug("pagename = " + pagename);
page.setName(pagename);
}
return page;
}
/**
* gets the pagename given the pagepath
* @param pagePath
* @return pagename
*/
protected String getPagename(String pagePath) {
String pageName = "";
if (hierarchyHandler == HierarchyHandler.DEFAULT ||
hierarchyHandler == HierarchyHandler.HIERARCHY_BUILDER) {
pageName = pagePath.substring(pagePath.lastIndexOf(File.separator) + 1);
} else if (hierarchyHandler == HierarchyHandler.PAGENAME_HIERARCHIES) {
String quotedSeparator = Pattern.quote(File.separator);
pageName = pagePath.replaceAll(quotedSeparator, CONFLUENCE_SEPARATOR);
}
return pageName;
}
/**
* This is where it all happens :). This method reads the files and runs the
* converters on the pages.
*
* @param pages The pages to be converted.
* @param converters The converters to run on the pages
* @return false if the user cancelled the task. Otherwise true.
*/
protected boolean convertPages(List<Page> pages, List<Converter> converters) {
boolean useUI = true;
String progressMessage = "Converting page files";
return convertPages(pages, converters, useUI, progressMessage);
}
/**
* @param pages pages to be converted
* @param converters converters to be run on pages
* @param useUI true if the UI should be updated
* @param progressMessage info provided to user describing progress monitor
* @return false if the user cancelled the task
*/
protected boolean convertPages(
List<Page> pages,
List<Converter> converters,
boolean useUI,
String progressMessage) {
boolean result = true;
if (useUI) JFrame.setDefaultLookAndFeelDecorated(true);
// Set up a progress monitor and progress count
ProgressMonitor progressMonitor = null;
int progress = 0;
long startTotalConvertTime = 0;
if (useUI) {
progressMonitor = createProgressMonitor(progressMessage, pages.size());
startTotalConvertTime = (new Date()).getTime();
}
// loop each page(file) through all the converters.
// We can't use the enhanced for loop here, because we need to
// remove pages that we can't read.
for (Iterator<Page> i = pages.iterator(); i.hasNext();) {
Page page = i.next();
long startTimeStamp = ((new Date()).getTime());
if (log.isInfoEnabled()) {
log.info("-------------------------------------");
log.info("converting page file: " + page.getName());
}
File file = page.getFile();
// String inputPage = null;
if (file == null) {
if (page.getOriginalText() != null && !"".equals(page.getOriginalText())) {
log.warn("This appears to be a unit test. Continue as for Unit Test.");
}
else {
log.warn("No file was set for page " + page.getName() + "! Skipping page.");
i.remove();
continue;
}
}
else if (page.getOriginalText() == null){
try {
String pageContents = FileUtils.readTextFile(file);
page.setOriginalText(pageContents);
} catch (IOException e) {
addError("Could not read file " + file.getAbsolutePath() + "! Skipping file.", e);
i.remove();
continue;
}
}
convertPage(converters, page);
if (log.isInfoEnabled()) {
long stopTimeStamp = ((new Date()).getTime());
log.info(" time to convert " + (stopTimeStamp - startTimeStamp) + "ms");
}
if (useUI) {
if (progressMonitor.isCanceled()) {
result = false;
break;
}
progressMonitor.setProgress(progress++);
if (progress % 10 == 0) {
progressMonitor.setNote("Converted " + progress + " out of " + pages.size() + " pages");
}
}
}
if (useUI) progressMonitor.close();
// deleteMe - start
// log.info("::: list of attachment paths not found:::");
// List allFilesNotFound = PmWikiPrepareAttachmentFilesConverter.allFilesNotFound;
// for (Object filesNotFound : allFilesNotFound) {
// String s = (String)filesNotFound;
// log.info("file not found::: "+s);
// }
// deleteMe - stop
if (useUI) {
long endTotalConvertTime = (new Date()).getTime();
long totalTimeToConvert = (endTotalConvertTime - startTotalConvertTime)/1000;
log.info("::: total time to convert files: "+ totalTimeToConvert+ " seconds.");
totalsFileLog.info(progressMessage+"::: total time to convert files: "+ totalTimeToConvert+ " seconds."+
"For "+pages.size()+" pages and using "+converters.size()+" converters.");
}
return result;
}
/**
* converts one page with the given converters
* @param converters list of converters
* @param page page object
*/
protected Page convertPage(List<Converter> converters, Page page) {
log.debug("pagename = " + page.getName());
if (page.getConvertedText() == null)
page.setConvertedText(page.getOriginalText()); //in case empty converter list
for (Converter converter : converters) {
try {
log.debug("running converter: "+converter.getKey());
converter.convert(page);
page.setOriginalText(page.getConvertedText());
} catch (Exception e) {
addError("Exception thrown by converter " + converter.getKey() +
" on page " + page.getName() + ". Continuing with next converter.", e);
}
}
return page;
}
/**
* Write pages to disk. They are saved to the directory output/output below the
* current working directory.
*
* @param pages The pages to save
* @param confSettings A settings object. The pattern (file name extension) is appended
* to each page name to create the file name.
*/
private void savePages(List<Page> pages, ConfluenceSettingsForm confSettings) {
FileUtils.createOutputDirIfNeeded();
// SimpleDateFormat sdf = new SimpleDateFormat("yy-MM-dd-HH-mm-ss");
// String outputDirName = "output" + File.separator + "output-" + sdf.format(new Date());
String outputDirName = "output" + File.separator + "output";
File outputDir = new File(outputDirName);
if (!outputDir.exists() && !outputDir.mkdir()) {
addError("Directory creation failed for directory " + outputDirName);
}
for (Page page : pages) {
String outputFileLoc = outputDirName + File.separator + page.getName() + confSettings.getPattern();
FileUtils.writeFile(page.getConvertedText(), outputFileLoc);
}
}
/**
* Puts up a dialog asking the user if the pages should be sent to Confluence.
* @return <code>True</code> if the user answers yet, otherwise <code>false</code>.
*/
private boolean okToSend() {
// We must use a final object here, since we'll be accessing it from within an inner class (the Runnable),
// but if we make a <code>final boolean</code>, the inner class can't change the value.
// Therefore we make a final array containing a single boolean. The inner class can't change the array,
// but changing <em>elements</em> of a final array is OK.
final boolean[] result = new boolean[]{false};
try {
// We need to use invokeAndWait(), because the converter engine is not run in the
// event dispatching thread.
SwingUtilities.invokeAndWait(new Runnable() {
public void run() {
int choice = JOptionPane.showInternalConfirmDialog(UWCForm2.getInstance().mainPanel,
"Do you want to send these pages to Confluence?", "information",
JOptionPane.YES_NO_OPTION, JOptionPane.QUESTION_MESSAGE);
result[0] = (choice == JOptionPane.YES_OPTION);
}
});
} catch (Exception ignored) {
// Do nothing -- in case of error, result[0] will be false and we will not continue.
}
return result[0];
}
/**
* Writes the pages to Confluence. If the process takes more than three seconds,
* a progress monitor will be displayed so that the user can see that something is
* indeed happening.
*
* @param pages The pages to output.
*/
private void writePages(List<Page> pages) {
ProgressMonitor progressMonitor = createProgressMonitor("Uploading page files", pages.size());
int progress = 0;
// at last write the pages to Confluence!
for (Page page : pages) {
sendPage(page, null);
if (progressMonitor.isCanceled()) {
break; // User pressed cancel -- leave the loop
}
progressMonitor.setProgress(progress++);
progressMonitor.setNote("Uploaded " + progress + " out of " + pages.size() + " pages.");
}
//attachedFiles is cleared so that if we do another conversion
//without closing the UWC, it won't think the attachment has already been
//attached
this.attachedFiles = null;
progressMonitor.close();
}
/**
* Writes a hierarchy of pages to Confluence. The empty nodes (those with page=null) will
* not be written if there already exists a page in Confluence. If there is no page at the
* corresponding place in Confluence, an empty page will be created.
*
* Like writePages(), this method will show a progress bar if the hierarchy takes more than
* a few seconds to send to Confluence.
*
* @param root The root of the hierarchy. Note: The root node itself will <strong>NOT</strong> be
* added to Confluence. All it's children will be added as top-level pages in the space.
*/
private void writeHierarchy(HierarchyNode root) {
// First count the number of nodes in the hierarchy so that we can set up the progress bar.
int numberOfNodes = root.countDescendants();
ProgressMonitor progressMonitor = createProgressMonitor("Uploading page files", numberOfNodes);
int progress = 0;
// at last write the pages to Confluence!
for (HierarchyNode topLevelPage : root.getChildren()) {
progress = writeHierarchy(topLevelPage, null, progressMonitor, progress);
if (progressMonitor.isCanceled()) {
break; // User pressed cancel -- leave the loop
}
}
progressMonitor.close();
}
/**
* This is the recursive part of <code>writeHierarchy</code>. Don't call this directly!
* Call writeHierarchy(root) instead.
*
* @param node The current node in the hierarchy
* @param parentId The Confluence "page ID" of the parent page
* @param progressMonitor A progress monitor (duh) for the users' benefit.
* @param progress The number of pages that have been converted so far
* (used to keep the progress monitor updated)
* @return The number of pages converted after this node and all its descendants have been added.
*/
private int writeHierarchy(HierarchyNode node, String parentId, ProgressMonitor progressMonitor, int progress) {
// First upload the page contained in this node
Page page = node.getPage();
if (page == null) {
// This node is a "placeholder" because there are pages further down the hierarchy but
// for some reason this node was not included in the conversion. Create an empty page.
// Note that this page will only be sent to Confluence if there was no page in place before.
page = new Page(null);
page.setName(node.getName());
page.setOriginalText("");
page.setConvertedText("");
}
String myId = sendPage(page, parentId);
progress++;
progressMonitor.setProgress(progress++);
progressMonitor.setNote("Uploaded " + progress + " out of " + progressMonitor.getMaximum() + " pages.");
// Then recursively upload all the node's descendants
if (node == null) {
log.error("Null node!");
}
for (HierarchyNode child : node.getChildren()) {
progress = writeHierarchy(child, myId, progressMonitor, progress);
if (progressMonitor.isCanceled()) {
break; // User pressed cancel -- leave the loop
}
}
return progress;
}
/**
* Sends a page and all its attachments to Confluence.
* If the page did not exist in Confluenc, it is created.
* If it already existed, it is updated.
* @param page A filled-in page object.
* @param parentId The ID of the page's parent page, or
* <code>null</code> if the page is a top-level page.
* @return The ID of the created or updated page in Confluence.
*/
private String sendPage(Page page, String parentId) {
// get a handle to the object that encapsulates writing pages
RemoteWikiBrokerOld rwb = RemoteWikiBrokerOld.getInstance();
PageForXmlRpcOld xmlrpcPage = new PageForXmlRpcOld();
xmlrpcPage.setTitle(page.getName());
xmlrpcPage.setContent(page.getConvertedText());
if (parentId != null) {
xmlrpcPage.setParentId(parentId);
}
xmlrpcPage = rwb.storeNewOrUpdatePage(xmlrpcPage);
String id = xmlrpcPage.getId();
// Send the attachments
for (File file : page.getAttachments()) {
AttachmentForXmlRpcOld attachment = new AttachmentForXmlRpcOld();
if (alreadyAttached(page, file) || tooBig(file) || doesNotExist(file))
continue;
attachment.setFileName(file.getName());
attachment.setFileLocation(file.getAbsolutePath());
attachment.setContentType(determineContentType(file));
attachment.setComment("Added by UWC, the Universal Wiki Converter");
try {
rwb.storeAttachment(xmlrpcPage.getId(), attachment);
} catch (IOException e) {
addError("Couldn't send attachment " +
file.getAbsolutePath() + "! Skipping attachment.", e);
} catch (XmlRpcException e) {
addError("Couldn't send attachment " +
file.getAbsolutePath() + "! Skipping attachment.", e);
}
}
return id;
}
/**
* @param file
* @return true if the given file does not exist on the filesystem.
*/
protected boolean doesNotExist(File file) {
boolean doesNotExist = !file.exists();
if (doesNotExist)
log.debug("File " + file.getName() + " does not exist: Skipping");
return doesNotExist;
}
/**
* @param file
* @return true is file size is too big
*/
protected boolean tooBig(File file) {
if (!file.exists()) return false;
int length = (int) file.length();
Properties properties = loadProperties(PROP_LOCATION);
String maxString = (String) properties.get(PROP_ATTACHMENT_SIZE_MAX);
int maxBytes = getAsBytes(maxString);
if (maxBytes < 0) return false;
boolean tooBig = length > maxBytes;
if (tooBig)
log.debug("File " + file.getName() + " is too big. Skipping.");
return tooBig;
}
/**
* @param fileLocation location of properties file
* @return a Properties object containing confluence settings
*/
protected Properties loadProperties(String fileLocation) {
/* most of this code grabbed from ConfluenceSettingsForm.populateConfluenceSettings*/
Properties properties = new Properties();
File confSettings = new File(fileLocation);
if (confSettings.exists()) {
// load properties file
FileInputStream fis;
try {
fis = new FileInputStream(fileLocation);
properties.load(fis);
fis.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
return properties;
}
/**
* @param maxString file size described as a String.
* Example: 5B, 5K, 5M, 5G, etc.
* @return as Bytes.
* Respectively: 5, 5120, 5242880, 5368709120
*/
protected int getAsBytes(String maxString) {
String maxRegex = "^(\\d+)(\\D)";
if (maxString == null || "".equals(maxString))
return -1;
int power, num = 0;
String numString, unitString = null;
if (Pattern.matches("^\\d+$", maxString)) {
unitString = "B";
numString = maxString;
}
else {
numString = maxString.replaceFirst(maxRegex, "$1");
unitString = maxString.replaceFirst(maxRegex, "$2");
}
try {
num = Integer.parseInt(numString);
} catch (NumberFormatException e) {
String message = PROP_ATTACHMENT_SIZE_MAX + " setting is malformed.\n" +
"Setting must be formatted like so: [number][unit], where unit is\n" +
"one of the following: B, K, M, G. No max attachment size set.";
log.error(message);
return -1;
}
unitString = unitString.toUpperCase();
char unit = unitString.toCharArray()[0]; //first char in that string
switch (unit) {
case ('B'): power = 0;break;
case ('K'): power = 1;break;
case ('M'): power = 2;break;
case ('G'): power = 3;break;
default: return -1;
}
int multiplier = (int) Math.pow(1024, power);
int value = num * multiplier;
return value;
}
/**
* @param page
* @param file
* @return true if a particular page already has a particular
* file attached.
*/
protected boolean alreadyAttached(Page page, File file) {
String pagename = page.getName();
String filename = file.getName();
String attachmentId = pagename + filename;
if (attachedFiles == null)
attachedFiles = new HashSet<String>();
boolean attached = attachedFiles.contains(attachmentId);
if (!attached) attachedFiles.add(attachmentId);
else log.debug("Attachment " + filename + " is already attached: Skipping.");
return attached;
}
/**
* Creates and initializes a progress monitor.
* @param heading A descriptio of the activity being monitored.
* @param numberOfPages The maximum to set in the progress monitor.
* @return The initialized progress monitor
*/
private ProgressMonitor createProgressMonitor(String heading, int numberOfPages) {
ProgressMonitor progressMonitor = new ProgressMonitor(
UWCForm2.getInstance().mainFrame, heading, "", 0, numberOfPages);
progressMonitor.setProgress(0);
progressMonitor.setMaximum(numberOfPages);
return progressMonitor;
}
/**
* This method determines the mime type of a file. It uses the file
* mime.types in the conf directory to map from the file name extension
* to a mime type. The mime type file should be read into the
* mimeTypes field before this method is called.
*
* @param file The file object
* @return the mime type of the file.
*/
public String determineContentType(File file) {
if (mimeTypes != null) {
return mimeTypes.getContentType(file);
} else {
// Assume it's an image
String filename = file.getName();
int extensionStart = filename.lastIndexOf(".");
if (extensionStart >= 0) {
String extension = filename.substring(extensionStart + 1);
log.info(extension + " --- " + filename);
return "image/" + extension;
}
// Hmm... No extension. Assume it's a text file.
return "text/plain";
}
}
/**
* append all of the errors into a single message automatically adding
* each with a new line
*
* @param errorMsg The error message
*/
public void addError(String errorMsg) {
log.error(errorMsg);
errorMessage += errorMsg + "\n";
}
/**
* Append all of the errors into a single message automatically adding
* a new line to each. This version of the method takes a Throwable as a
* second argument. The Throwable is output to the log object, but is not
* placed in the error message.
*
* @param errorMsg The error message
* @param e An exception.
*/
public void addError(String errorMsg, Throwable e) {
log.error(errorMsg, e);
errorMessage += errorMsg + "\n";
}
/**
* retrieve the error messages and also clear them out
*
* @return errors
*/
public String getErrorMessage() {
String errorsTemp = errorMessage;
// when we retrieve the errors we clear them for the next round
errorMessage = "";
return errorsTemp;
}
Pattern switchPattern = Pattern.compile("switch");
Pattern suffixPattern = Pattern.compile("suffix");
private boolean handlingPageHistories = false;
private String pageHistorySuffix = null;
/**
* set the page history state to reflect the page history property
* and associated value that are passed as arguments
* @param key
* @param value
*/
protected void handlePageHistoryProperty(String key, String value) {
Matcher switchFinder = switchPattern.matcher(key);
if (switchFinder.find()) {
//the default should be false, so it's ok to just parse the string.
this.handlingPageHistories = Boolean.parseBoolean(value);
return;
}
Matcher suffixFinder = suffixPattern.matcher(key);
if (suffixFinder.find()) {
setPageHistorySuffix(value);
return;
}
}
/**
* @param key
* @return true if the given key is the switch to turn on the
* Hierarchy framework
*/
protected boolean isHierarchySwitch(String key) {
Matcher switchFinder = switchPattern.matcher(key);
return switchFinder.find();
}
/**
* determines if the given string represents an allowed
* non converter property: (hierarchy builder or page history preserver)
* @param input represents an entire converter/property string. For example:
* <br/>
* Wiki.0011.somefilename.propertytype=something
* @return true if it's an expected/allowed non converter property
*/
public boolean isNonConverterProperty(String input) {
String converterTypes =
"(" +
"(" +
NONCONVERTERTYPE_HIERARCHYBUILDER +
")" +
"|" +
"(" +
NONCONVERTERTYPE_PAGEHISTORYPRESERVATION +
")" +
")";
String converterPattern = "[-\\w\\d.]+?" + converterTypes + "=" + ".*";
return input.matches(converterPattern);
}
/**
* @return true if the converter should handle page histories
*/
public boolean isHandlingPageHistories() {
return this.handlingPageHistories;
}
/**
* @return the current page history suffix
*/
public String getPageHistorySuffix() {
return this.pageHistorySuffix;
}
/**
* sorts the given list of pages.
* Note: sorting will take into account page name
* and page version. Non unique page objects will be culled.
* @param pages list of Page objects
* @return sorted list
*/
protected List<Page> sortByHistory(List<Page> pages) {
List<Page> sortedPages = new ArrayList<Page>();
Set<Page> sorted = new TreeSet<Page>();
sorted.addAll(pages); //sort them and get rid of non-unique pages
sortedPages.addAll(sorted); //turn them back into a list
return sortedPages;
}
Pattern hashPattern = Pattern.compile("#+");
/**
* sets the page history suffix, if it's a valid suffix.
* If not, sets it to null.
* @param suffix candidate suffix, a valid candidate will have
* a numeric component, represented by a '#' (hash) symbol
* <br/>
* Example: -v#.txt
* @return true, if a valid suffix was saved.
* false, if the suffix was invalid, and therefore was not saved.
*/
protected boolean setPageHistorySuffix(String suffix) {
//check for suffix goodness
Matcher hashFinder = hashPattern .matcher(suffix);
if (hashFinder.find()) {
this.pageHistorySuffix = suffix;
return true;
}
log.error("Error trying to preserve page history: Suffix '" + suffix + "' does not have a sortable component. Must include '#'.");
this.pageHistorySuffix = null;
return false;
}
protected HierarchyBuilder getHierarchyBuilder() {
return hierarchyBuilder;
}
protected HierarchyHandler getHierarchyHandler() {
return hierarchyHandler;
}
/**
* sets how the hierarchy framework is to be used.
* @param input "UseBuilder", "UsePagenames", or "Default".
* If input is none of these, no changes occur
*/
private void setHierarchyHandler(String input) {
if (input.matches("UseBuilder")) hierarchyHandler = HierarchyHandler.HIERARCHY_BUILDER;
else if (input.matches("UsePagenames")) hierarchyHandler = HierarchyHandler.PAGENAME_HIERARCHIES;
else if (input.matches("Default")) hierarchyHandler = HierarchyHandler.DEFAULT;
}
/**
* sets the location of the confluenceSettings.properties file.
* Handy for unit testing
* @param prop_location
*/
protected void setPropLocation(String prop_location) {
PROP_LOCATION = prop_location;
}
}