| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.nutch.tools; |
| |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.Serializable; |
| import java.util.Properties; |
| |
| public class CommonCrawlConfig implements Serializable { |
| |
| /** |
| * Serial version UID |
| */ |
| private static final long serialVersionUID = 5235013733207799661L; |
| |
| // Prefix for key value in the output format |
| private String keyPrefix = ""; |
| |
| private boolean simpleDateFormat = false; |
| |
| private boolean jsonArray = false; |
| |
| private boolean reverseKey = false; |
| |
| private String reverseKeyValue = ""; |
| |
| private boolean compressed = false; |
| |
| private long warcSize = 0; |
| |
| private String outputDir; |
| |
| /** |
| * Default constructor |
| */ |
| public CommonCrawlConfig() { |
| // TODO init(this.getClass().getResourceAsStream("CommonCrawlConfig.properties")); |
| } |
| |
| public CommonCrawlConfig(InputStream stream) { |
| init(stream); |
| } |
| |
| private void init(InputStream stream) { |
| if (stream == null) { |
| return; |
| } |
| Properties properties = new Properties(); |
| |
| try { |
| properties.load(stream); |
| } catch (IOException e) { |
| // TODO |
| } finally { |
| try { |
| stream.close(); |
| } catch (IOException e) { |
| // TODO |
| } |
| } |
| |
| setKeyPrefix(properties.getProperty("keyPrefix", "")); |
| setSimpleDateFormat(Boolean.parseBoolean(properties.getProperty("simpleDateFormat", "False"))); |
| setJsonArray(Boolean.parseBoolean(properties.getProperty("jsonArray", "False"))); |
| setReverseKey(Boolean.parseBoolean(properties.getProperty("reverseKey", "False"))); |
| } |
| |
| public void setKeyPrefix(String keyPrefix) { |
| this.keyPrefix = keyPrefix; |
| } |
| |
| public void setSimpleDateFormat(boolean simpleDateFormat) { |
| this.simpleDateFormat = simpleDateFormat; |
| } |
| |
| public void setJsonArray(boolean jsonArray) { |
| this.jsonArray = jsonArray; |
| } |
| |
| public void setReverseKey(boolean reverseKey) { |
| this.reverseKey = reverseKey; |
| } |
| |
| public void setReverseKeyValue(String reverseKeyValue) { |
| this.reverseKeyValue = reverseKeyValue; |
| } |
| |
| public String getKeyPrefix() { |
| return this.keyPrefix; |
| } |
| |
| public boolean getSimpleDateFormat() { |
| return this.simpleDateFormat; |
| } |
| |
| public boolean getJsonArray() { |
| return this.jsonArray; |
| } |
| |
| public boolean getReverseKey() { |
| return this.reverseKey; |
| } |
| |
| public String getReverseKeyValue() { |
| return this.reverseKeyValue; |
| } |
| |
| public boolean isCompressed() { |
| return compressed; |
| } |
| |
| public void setCompressed(boolean compressed) { |
| this.compressed = compressed; |
| } |
| |
| public long getWarcSize() { |
| return warcSize; |
| } |
| |
| public void setWarcSize(long warcSize) { |
| this.warcSize = warcSize; |
| } |
| |
| public String getOutputDir() { |
| return outputDir; |
| } |
| |
| public void setOutputDir(String outputDir) { |
| this.outputDir = outputDir; |
| } |
| } |