| # |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| # |
| |
| ########################### |
| # Configuration for Flume |
| ########################## |
| # TODO: Should be moved into the flume folder, but its a little bit of a pain in the assembly configuration |
| # for now, just leaving it here - jyates |
| # |
| # configurationforagent |
| agent.sources=spooling-source |
| agent.sinks=phoenix-sink |
| agent.channels=memoryChannel |
| # configurationforchannel |
| agent.channels.memoryChannel.type=memory |
| agent.channels.memoryChannel.transactionCapacity=100 |
| agent.channels.memoryChannel.byteCapacityBufferPercentage=20 |
| # configurationforsource |
| agent.sources.spooling-source.type=spooldir |
| agent.sources.spooling-source.channels=memoryChannel |
| agent.sources.spooling-source.spoolDir=/opt/logs |
| # configurationforinterceptor |
| agent.sources.spooling-source.interceptors=i1 |
| agent.sources.spooling-source.interceptors.i1.type=host |
| agent.sources.spooling-source.interceptors.i1.hostHeader=f_host |
| # configurationforsink |
| agent.sinks.phoenix-sink.type=org.apache.phoenix.flume.sink.PhoenixSink |
| agent.sinks.phoenix-sink.channel=memoryChannel |
| agent.sinks.phoenix-sink.batchSize=100 |
| agent.sinks.phoenix-sink.table=APACHE_LOGS |
| agent.sinks.phoenix-sink.ddl=CREATETABLEIFNOTEXISTSAPACHE_LOGS(uidVARCHARNOTNULL,hostVARCHAR,identityVARCHAR,userVARCHAR,timeVARCHAR,methodVARCHAR,requestVARCHAR,protocolVARCHAR,statusINTEGER,sizeINTEGER,refererVARCHAR,agentVARCHAR,f_hostVARCHARCONSTRAINTpkPRIMARYKEY(uid)) |
| agent.sinks.phoenix-sink.zookeeperQuorum=localhost |
| agent.sinks.phoenix-sink.serializer=REGEX |
| agent.sinks.phoenix-sink.serializer.rowkeyType=uuid |
| agent.sinks.phoenix-sink.serializer.regex=([^]*)([^]*)([^]*)(-|\\[[^\\]]*\\])\"([^]+)([^]+)([^\"]+)\"(-|[0-9]*)(-|[0-9]*)(?:([^\"]*|\"[^\"]*\")([^\"]*|\"[^\"]*\"))? |
| agent.sinks.phoenix-sink.serializer.columns=host,identity,user,time,method,request,protocol,status,size,referer,agent |
| agent.sinks.phoenix-sink.serializer.headers=f_host |
| |
