Add source for website
diff --git a/.gitignore b/.gitignore
index a13ec8f..a83d1b4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,4 +27,10 @@
*.class
# apache RAT
-.rat-excludes
\ No newline at end of file
+.rat-excludes
+
+# website output
+website/output
+website/tmp
+website/.sass-cache
+website/crash.log
\ No newline at end of file
diff --git a/website/Gemfile b/website/Gemfile
new file mode 100644
index 0000000..911f923
--- /dev/null
+++ b/website/Gemfile
@@ -0,0 +1,3 @@
+source :rubygems
+
+gem "compass"
\ No newline at end of file
diff --git a/website/README.markdown b/website/README.markdown
new file mode 100644
index 0000000..19931d0
--- /dev/null
+++ b/website/README.markdown
@@ -0,0 +1,29 @@
+Code for the S4 website
+
+This website uses the [nanoc](http://nanoc.stoneship.org/) static website generator
+
+Entry pages are written with haml and the documentation is written with markdown.
+
+
+# To compile the site:
+
+* Install nanoc: `gem install nanoc`
+* `nanoc compile`
+
+The generated static website is in `output/`
+
+# To upload the site to apache
+
+## first, commit the generated website to svn
+
+ cp -R output/* $S4_SVN_LOC/site
+ cd $S4_SVN_LOC
+ svn update
+ svn status
+ svn add <whatever is missing>
+ svn commit --username <apache username> -m "commit message"
+
+## then checkout into web server
+ ssh people.apache.org
+ cd /www/incubator.apache.org/content/s4
+ svn checkout http://svn.apache.org/repos/asf/incubator/s4/site .
\ No newline at end of file
diff --git a/website/Rakefile b/website/Rakefile
new file mode 100644
index 0000000..f484d24
--- /dev/null
+++ b/website/Rakefile
@@ -0,0 +1 @@
+require 'nanoc3/tasks'
\ No newline at end of file
diff --git a/website/Rules b/website/Rules
new file mode 100644
index 0000000..d69f9be
--- /dev/null
+++ b/website/Rules
@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+
+# A few helpful tips about the Rules file:
+#
+# * The order of rules is important: for each item, only the first matching
+# rule is applied.
+#
+# * Item identifiers start and end with a slash (e.g. “/about/” for the file
+# “content/about.html”). To select all children, grandchildren, … of an
+# item, use the pattern “/about/*/”; “/about/*” will also select the parent,
+# because “*” matches zero or more characters.
+
+require 'compass'
+
+Compass.add_project_configuration 'compass/config.rb' # when using Compass 0.10
+
+compile '/style/*/' do
+ filter :sass, Compass.sass_engine_options.merge(:syntax => :scss)
+end
+
+compile '/images/*/' do
+ # do nothing
+end
+
+compile '/doc/*' do
+ if item.binary?
+ # don’t filter binary items
+ else
+ filter :kramdown
+ layout 'default'
+ end
+end
+
+compile '*' do
+ if item[:extension] == "haml"
+ filter :haml
+ layout 'default'
+ end
+ filter :relativize_paths, :type => :html
+end
+
+route '/style/partials/*/' do
+ # don't output partials, so return nil
+ nil
+end
+
+route '/style/*/' do
+ # don't generate a directory like we do for HTML files
+ item.identifier.chop + '.css'
+end
+
+route '/images/*/' do
+ # don't generate a directory like we do for HTML files
+ item.identifier.chop + '.' + item[:extension]
+end
+
+route '*' do
+ item.identifier + 'index.html'
+end
+
+layout '*', :haml
diff --git a/website/compass/config.rb b/website/compass/config.rb
new file mode 100644
index 0000000..39317a1
--- /dev/null
+++ b/website/compass/config.rb
@@ -0,0 +1,10 @@
+http_path = "/"
+project_path = "."
+css_dir = "output/style"
+sass_dir = "content/style"
+images_dir = "output/images"
+
+# when using SCSS:
+sass_options = {
+ :syntax => :scss
+}
diff --git a/website/config.yaml b/website/config.yaml
new file mode 100644
index 0000000..243e1aa
--- /dev/null
+++ b/website/config.yaml
@@ -0,0 +1,44 @@
+# A list of file extensions that nanoc will consider to be textual rather than
+# binary. If an item with an extension not in this list is found, the file
+# will be considered as binary.
+text_extensions: [ 'css', 'erb', 'haml', 'htm', 'html', 'js', 'less', 'markdown', 'md', 'php', 'rb', 'sass', 'scss', 'txt', 'xhtml', 'xml' ]
+
+# The path to the directory where all generated files will be written to. This
+# can be an absolute path starting with a slash, but it can also be path
+# relative to the site directory.
+output_dir: output
+
+# A list of index filenames, i.e. names of files that will be served by a web
+# server when a directory is requested. Usually, index files are named
+# “index.hml”, but depending on the web server, this may be something else,
+# such as “default.htm”. This list is used by nanoc to generate pretty URLs.
+index_filenames: [ 'index.html' ]
+
+# Whether or not to generate a diff of the compiled content when compiling a
+# site. The diff will contain the differences between the compiled content
+# before and after the last site compilation.
+enable_output_diff: false
+
+# The data sources where nanoc loads its data from. This is an array of
+# hashes; each array element represents a single data source. By default,
+# there is only a single data source that reads data from the “content/” and
+# “layout/” directories in the site directory.
+data_sources:
+ -
+ # The type is the identifier of the data source. By default, this will be
+ # `filesystem_unified`.
+ type: filesystem_unified
+
+ # The path where items should be mounted (comparable to mount points in
+ # Unix-like systems). This is “/” by default, meaning that items will have
+ # “/” prefixed to their identifiers. If the items root were “/en/”
+ # instead, an item at content/about.html would have an identifier of
+ # “/en/about/” instead of just “/about/”.
+ items_root: /
+
+ # The path where layouts should be mounted. The layouts root behaves the
+ # same as the items root, but applies to layouts rather than items.
+ layouts_root: /
+
+google_analytics_account_id: UA-19490961-1
+google_analytics_domain: .s4.io
diff --git a/website/content/contrib.haml b/website/content/contrib.haml
new file mode 100644
index 0000000..a97bb0b
--- /dev/null
+++ b/website/content/contrib.haml
@@ -0,0 +1,74 @@
+---
+title: Contribute
+---
+#main{:class=>"span-18 colborder"}
+ / collaboration
+ %h2{:class=>"secheader"}= "collaboration model"
+ %p= "The project is maintain by volunteers and is open to anybody interested in contributing to the project. To contribute, you can write code, review code, report bugs, suggest ideas, propose design changes, write documentation, answer questions in the mailing list, and much more. The more you contribute the more you can make a difference."
+
+ / version control
+ %h2{:class=>"secheader"}= "version control"
+
+ %p= "S4 sources are available through <b>git</b>. Most development is done on the <i>piper</i> branch"
+
+ %p= "See a summary <a href='https://git-wip-us.apache.org/repos/asf?p=incubator-s4.git;a=summary'>here.</a>"
+
+
+ / access to the repository
+ %h2{:class=>"secheader"}= "access to the repository"
+ %ul
+ %li= "as a committer: <pre>git clone https://user_name@git-wip-us.apache.org/repos/asf/incubator-s4.git</pre>"
+ %li= "as a user or contributor: <pre>git clone http://git-wip-us.apache.org/repos/asf/incubator-s4.git</pre> (though since June 2012, you may have to try with <b>https</b> :<pre>git clone https://git-wip-us.apache.org/repos/asf/incubator-s4.git</pre> , even though this is not reflected yet on the Apache git documentation)."
+ %ul
+
+ %p= "For more information, have a look there: <a href='https://git-wip-us.apache.org/'>https://git-wip-us.apache.org/</a>"
+
+
+ / eclipse project
+ %h2{:class=>"secheader"}= "creating an eclipse project"
+ %p= "type: <pre>./gradlew eclipse</pre> You can now import the project into Eclipse. To learn more, visit the <b><a href='http://www.gradle.org'>Gradle website</a></b>."
+
+ / submission guidelines
+ %h2{:class=>"secheader"}= "submission guidelines"
+
+ %p="Before you start, send a message to the S4 developer <b><a href='https://cwiki.apache.org/confluence/display/S4/S4+Apache+mailing+lists'>mailing list</a></b>, or file a bug report in <b><a href='https://issues.apache.org/jira/browse/S4'>Jira</a></b>. Describe your proposed changes and check that they fit in with what others are doing and have planned for the project."
+
+ / modifying the source code
+ %h2{:class=>"secheader"}= "modifying the source code"
+
+ %p="Here are a few things to keep in mind:"
+ %ol
+ %li="All public classes, interfaces, and methods should have informative <b><a href='http://www.oracle.com/technetwork/java/javase/documentation/index-137868.html'>Javadoc comments</a></b>."
+ %ul
+ %li= "BAD: <span class='high'>getCondensedLength()</span>: Gets condensed length."
+ %li= "BETTER: <span class='high'>getCondensedLength()</span>: Number of characters in the string, excluding non alpha-numeric characters."
+ %li="Do not use @author tags."
+ %li="Code should be written according to <b><a href='http://www.oracle.com/technetwork/java/codeconv-138413.html'>Java\'s conventions</a></b>."
+ %li="Formatting: (from S4 0.5, see \"automatic formatting\" below)"
+ %ul
+ %li="Indent four spaces per level, not two or six or eight, etc... four."
+ %li="No tabs for indentation, spaces only."
+ %li="Contributions should pass existing unit tests."
+ %li="New unit tests should be provided to demonstrate bugs and fixes. <b><a href='http://www.junit.org'>Junit</a></b> is our test framework."
+
+ / automatic source code formatting
+ %h2{:class=>"secheader"}= "automatic source code formatting"
+
+ %p="You must configure your IDE to follow the formatting guidelines. This is needed to get clean diffs."
+
+ %p="To automatically format code in Eclipse:"
+ %ol
+ %li="preferences --> Java --> Code style --> Formatter , then import s4/config/eclipse/s4-eclipse-format.xml"
+ %li="preferences --> Java --> Editor --> Save actions --> format source code, format all lines"
+
+ %p="If you don't use Eclipse, you may still use Eclipse's code formatter in a <a href='http://www.peterfriese.de/formatting-your-code-using-the-eclipse-code-formatter/'>standalone mode.</a>"
+
+#contrib_sidebar{:class=>"span-5 last"}
+ #participate
+ %h2{:class=>"secheader"}= "participate"
+ %ul{:class=>'large greybar'}
+ %li= "Join the <a href='https://cwiki.apache.org/confluence/display/S4/S4+Apache+mailing+lists' onClick=\"#{google_analytics_event("External", "Forum", "Contrib 2 - https://cwiki.apache.org/confluence/display/S4/S4+Apache+mailing+lists")}\">forum</a>"
+
+ #twitter_follow
+ %a{:href=>"http://www.twitter.com/s4project", :onClick=> google_analytics_event("External", "Twitter", "Contrib - http://www.twitter.com/s4project")}
+ %img{:src=>"http://twitter-badges.s3.amazonaws.com/follow_us-b.png", :alt=>"Follow s4project on Twitter"}
diff --git a/website/content/doc/0.6.0/application_dependencies.md b/website/content/doc/0.6.0/application_dependencies.md
new file mode 100644
index 0000000..d1d9ca3
--- /dev/null
+++ b/website/content/doc/0.6.0/application_dependencies.md
@@ -0,0 +1,49 @@
+---
+title: Adding application dependencies
+---
+
+> Make sure you have already read the [walkthrough](../walkthrough)
+
+# How to add dependencies to my S4 application?
+
+Your application typically depends on various external libraries. Here is how to configure those dependencies in an S4 project. We assume here that you are working with a sample project automatically generated through the `s4 newApp` script.
+
+## Dependencies on public artifacts
+
+* Add maven artifacts definitions to the gradle build file. For instance, add twitter4j_core and twitter4j_stream:
+
+
+ project.ext["libraries"] = [
+ twitter4j_core: 'org.twitter4j:twitter4j-core:2.2.5',
+ twitter4j_stream: 'org.twitter4j:twitter4j-stream:2.2.5',
+ s4_base: 'org.apache.s4:s4-base:0.5.0',
+ s4_comm: 'org.apache.s4:s4-comm:0.5.0',
+ s4_core: 'org.apache.s4:s4-core:0.5.0'
+ ]
+
+* Add these dependencies as compile-time dependencies. For instance:
+
+ dependencies {
+ compile (libraries.s4_base)
+ compile (libraries.s4_comm)
+ compile (libraries.s4_core)
+ compile (libraries.twitter4j_core)
+ compile (libraries.twitter4j_stream)
+ }
+
+* If you use an IDE such as eclipse, you may update your project's classpath with: `./gradlew eclipse`
+
+A good source for finding dependencies is for instance [http://search.maven.org/](http://search.maven.org/) where you also get the syntax for gradle scripts (see grails syntax).
+
+
+>The application dependencies will be automatically included in the s4r archive that you create and publish.
+
+
+## Dependencies on non-public artifacts
+
+You may have dependencies that are not published to maven repositories. In that case you should either:
+
+* publish them to your local maven repository, see [http://maven.apache.org/guides/mini/guide-3rd-party-jars-local.html]([http://maven.apache.org/guides/mini/guide-3rd-party-jars-local.html)
+* add them to the _lib_ directory
+
+In both cases you still have to declare them as compile-time dependencies.
\ No newline at end of file
diff --git a/website/content/doc/0.6.0/configuration.md b/website/content/doc/0.6.0/configuration.md
new file mode 100644
index 0000000..59ffd26
--- /dev/null
+++ b/website/content/doc/0.6.0/configuration.md
@@ -0,0 +1,158 @@
+---
+title: Configuration
+---
+
+# Toolset
+
+S4 provides a set of tools to:
+
+* define S4 clusters: `s4 newCluster`
+* start S4 nodes: `s4 node`
+* package applications: `s4 s4r`
+* deploy applications: `s4 deploy`
+* start a Zookeeper server for easy testing: `s4 zkServer`
+ * `s4 zkServer -t` will start a Zookeeper server and automatically configure 2 clusters
+* view the status of S4 clusters coordinated by a given Zookeeper ensemble: `s4 status`
+
+
+ ./s4
+
+will give you a list of available commands.
+
+ ./s4 <command> -help
+
+will provide detailed documentation for each of these commands.
+
+
+# Cluster configuration
+
+Before starting S4 nodes, you must define a logical cluster by specifying:
+
+* a name for the cluster
+* a number of partitions (~ tasks)
+* an initial port number for listener sockets
+ * you must specify a free port, considering that each of the nodes of the cluster will open a different port, with a number monotonically increasing from the initial number. For instance, for a cluster of 10 nodes and an initial port 12000, ports 12000 to 12009 will be used among the nodes.
+ * those ports are used for inter node communication.
+
+
+The cluster configuration is maintained in Zookeeper, and can be set using S4 tools:
+
+ ./s4 newCluster -c=cluster1 -nbTasks=2 -flp=12000
+See tool documentation by typing:
+
+ ./s4 newCluster -help
+
+
+# Node configuration
+
+*Platform* *code and* *application* *code are fully configurable,* *at deployment time{*}*.*
+
+S4 nodes start as simple *bootstrap* processes whose initial role is merely to connect the cluster manager:
+
+* the bootstrap code connects to the cluster manager
+* when an application is available on the cluster, the node gets notified
+* it downloads the platform configuration and code, as specified in the configuration of the deployed application.
+* the communication and core components are loaded, bound and initialized
+* the application configuration and code, as specified in the configuration of the deployed applciation, is downloaded
+* the application is initialized and started
+
+This figure illustrates the separation between the bootstrap code, the S4 platform code, and application code in an S4 node:
+
+
+
+
+
+Therefore, for starting an S4 node on a given host, you only need to specify:
+
+* the connection string to the cluster management system (Zookeeper) `localhost:2181` by default
+* the name of the logical cluster to which this node will belong
+
+Example:
+`./s4 node -c=cluster1 -zk=host.domain.com`
+
+
+# Application configuration
+
+Deploying applications is easier when we can define both the parameters of the application *and* the target environment.
+
+In S4, we achieve this by specifying *both* application parameters and S4 platform parameters in the deployment phase :
+
+* which application class to use
+* where to fetch application code
+* which specific modules to use
+* where to fetch these modules
+* string configuration parameters - that can be used by the application and the modules
+
+
+
+## Modules configuration
+
+S4 follows a modular design and uses[Guice](http://code.google.com/p/google-guice/) for defining modules and injecting dependencies.
+
+As illustrated above, an S4 node is composed of:
+* a base module that specifies how to connect to the cluster manager and how to download code
+* a communication module that specifies communication protocols, event listeners and senders
+* a core module that specifies the deployment mechanism, serialization mechanism
+* an application
+
+### default parameters
+
+For the [comm module](https://github.com/apache/incubator-s4/blob/dev/subprojects/s4-comm/src/main/resources/default.s4.comm.properties): communication protocols, tuning parameters for sending events
+
+For the core module, there is no default parameters.
+
+### overriding modules
+
+We provide default modules, but you may directly specify others through the command line, and it is also possible to override them with new modules and even specify new ones (custom modules classes must provide an empty no-args constructor).
+
+Custom overriding modules can be specified when deploying the application, through the`deploy` command, through the _emc_ or _modulesClasses_ option.
+
+For instance, in order to enable file system based checkpointing, pass the corresponding checkpointing module class :
+
+ ./s4 deploy -s4r=uri/to/app.s4r -c=cluster1 -appName=myApp \
+ -emc=org.apache.s4.core.ft.FileSystemBackendCheckpointingModule
+
+You can also write your own custom modules. In that case, just package them into a jar file, and specify how to fetch that file when deploying the application, with the _mu_ or _modulesURIs_ option.
+
+For instance, if you checkpoint through a specific key value store, you can write you own checkpointing implementation and module, package that into fancyKeyValueStoreCheckpointingModule.jar , and then:
+
+ ./s4 node -c=cluster1 -emc=my.project.FancyKeyValueStoreBackendCheckpointingModule \
+ -mu=uri/to/fancyKeyValueStoreCheckpointingModule.jar
+
+### overriding parameters
+
+A simple way to pass parameters to your application code is by:
+
+* injecting them in the application class:
+
+ @Inject
+ @Named('myParam')
+ param
+* specifying the parameter value at node startup (using -p inline with the node command, or with the '@' syntax)
+
+S4 uses an internal Guice module that automatically injects configuration parameters passed through the deploy command to matching `@Named` parameters.
+
+Both application and platform parameters can be overriden. For instance, specifying a custom storage path for the file system based checkpointing mechanism would be passing the `s4.checkpointing.filesystem.storageRootPath` parameter:
+
+ ./s4 deploy -s4r=uri/to/app.s4r -c=cluster1 -appName=myApp \
+ -emc=org.apache.s4.core.ft.FileSystemBackendCheckpointingModule \
+ -p=s4.checkpointing.filesystem.storageRootPath=/custom/path
+
+## File-based configuration
+
+Instead of specifying node parameters inline, you may refer to a file with the '@' notation:
+./s4 deploy @/path/to/config/file
+With contents of the referenced file like:
+
+ -s4r=uri/to/app.s4r
+ -c=cluster1
+ -appName=myApp
+ -emc=org.apache.s4.core.ft.FileSystemBackendCheckpointingModule
+ -p=param1=value1,param2=value2
+
+
+
+
+## Logging
+
+S4 uses [logback](http://logback.qos.ch/), and [here](https://git-wip-us.apache.org/repos/asf?p=incubator-s4.git;a=blob_plain;f=subprojects/s4-core/src/main/resources/logback.xml;h=ea8c85a104b475f1b9dea641656e76eb3b6a9d4c;hb=piper) is the default configuration file. You may tweak this configuration by adding your own logback.xml file in the `lib/` directory (for a binary release) or in the `subprojects/s4-tools/build/install/s4-tools/lib/` directory (for a source release or checkout from git).
\ No newline at end of file
diff --git a/website/content/doc/0.6.0/dev_tips.md b/website/content/doc/0.6.0/dev_tips.md
new file mode 100644
index 0000000..8751a3a
--- /dev/null
+++ b/website/content/doc/0.6.0/dev_tips.md
@@ -0,0 +1,45 @@
+---
+title: Development tips
+---
+
+Here are a few tips to ease the development of S4 applications.
+
+
+### Import an S4 project into your IDE
+
+You can run `gradlew eclipse` or `gradlew idea` at the root of your S4 application directory. Then simply import the project into eclipse or intellij. You'll have both your application classes _and_ S4 libraries imported to the classpath of the project.
+
+In order to get the transitive dependencies of the platform included as well, you should:
+
+* Download a source distribution
+* Install S4 and its dependencies in your local maven repository
+
+ // from s4 source distribution root directory
+ ./gradlew install -DskipTests
+* Then run `gradlew eclipse` or `gradlew idea`
+
+
+
+
+### Start a local Zookeeper instance
+
+* Use the default test configuration (2 clusters with following configs: `c=testCluster1:flp=12000:nbTasks=1` and `c=testCluster2:flp=13000:nbTasks=1`)
+
+ s4 zkServer -t
+* Start a Zookeeper instance with your custom configuration, e.g. with 1 partition:
+
+ s4 zkServer -clusters=c=testCluster1:flp=12000:nbTasks=1
+
+
+### Load an application in a new node directly from an IDE
+
+This allows to *skip the packaging phase!*
+
+A requirement is that you have both the application classes and the S4 classes in your classpath. See above.
+
+Then you just need to run the `org.apache.s4.core.Main` class and pass:
+
+* the cluster name: `-c=testCluster1`
+* the app class name: `-appClass=myAppClass`
+
+If you use a local Zookeeper instance, there is no need to specify the `-zk` option.
\ No newline at end of file
diff --git a/website/content/doc/0.6.0/event_dispatch.md b/website/content/doc/0.6.0/event_dispatch.md
new file mode 100644
index 0000000..ee61b23
--- /dev/null
+++ b/website/content/doc/0.6.0/event_dispatch.md
@@ -0,0 +1,77 @@
+---
+title: Event dispatch
+---
+
+Events are dispatched according to their key.
+
+The key is identified in an `Event` through a `KeyFinder`.
+
+Dispatch can be configured for:
+* dispatching events to partitions (_outgoing dispatch_)
+* dispatching external events within a partition (_incoming dispatch_)
+
+# Outgoing dispatch
+
+A stream can be defined with a KeyFinder, as :
+
+Stream<TopicEvent> topicSeenStream = createStream("TopicSeen", new KeyFinder<TopicEvent>() {
+
+ @Override
+ public List<String> get(final TopicEvent arg0) {
+ return ImmutableList.of(arg0.getTopic());
+ }
+ }, topicCountAndReportPE);
+
+
+When an event is sent to the "TopicSeen" stream, its key will be identified through the KeyFinder implementation, hashed and dispatched to the matching partition.
+
+
+The same logic applies when defining _output streams_.
+
+If we use an AdapterApp subclass, the `remoteStreamKeyFinder` should be defined in the `onInit()` method, _before_ calling `super.onInit()`:
+
+ @Override
+ protected void onInit() {
+ ...
+ remoteStreamKeyFinder = new KeyFinder<Event>() {
+
+ @Override
+ public List<String> get(Event event) {
+ return ImmutableList.of(event.get("theKeyField"));
+ }
+ };
+ super.onInit()
+ ...
+
+
+If we use a standard App, we use the `createOutputStream(String name, KeyFinder<Event> keyFinder)` method.
+
+
+bq. If the KeyFinder is not defined for the output streams, events are sent to partitions of the connected cluster in a round robin fashion.
+
+
+# Incoming dispatch from external events
+
+When receiving events from a remote application, we _must_ define how external events are dispatched internally, to which PEs and based on which keys. For that purpose, we simply define and _input stream_ with the corresponding KeyFinder:
+
+createInputStream("names", new KeyFinder<Event>() {
+
+ @Override
+ public List<String> get(Event event) {
+ return Arrays.asList(new String[] { event.get("name") });
+ }
+ }, helloPE);
+
+
+In this case, a name is extracted from each event, the PE instance with this key is retrieved or created, and the event sent to that instance.
+
+
+Alternatively, we can use a unique PE instance for processing events in a given node. For that we simply define the input stream without a KeyFinder, _and_ use a singleton PE:
+
+ HelloPE helloPE = createPE(HelloPE.class);
+ helloPE.setSingleton(true);
+ createInputStream("names", helloPE);
+
+
+In this case, all events will be dispatched to the only HelloPE instance in this partition, regardless of the content of the event.
+
diff --git a/website/content/doc/0.6.0/fault_tolerance.md b/website/content/doc/0.6.0/fault_tolerance.md
new file mode 100644
index 0000000..241374c
--- /dev/null
+++ b/website/content/doc/0.6.0/fault_tolerance.md
@@ -0,0 +1,183 @@
+---
+title: Fault tolerance
+---
+Stream processing applications are typically long running applications, and they may accumulate state over extended periods of time.
+
+
+Running a distributed system over a long period of time implies there will be:
+
+
+- failures
+- infrastructure updates
+- scheduled restarts
+- application updates
+
+
+In each of these situations, some or all of S4 nodes will be shutdown. The system may therefore be partly unavailable, and in-memory state accumulated during the execution may be lost.
+
+
+In order to deal with this kind of situation, S4 provides:
+
+
+- high availability
+- state recovery (based on checkpointing)
+- while preserving low processing latency
+
+
+In this document, we first describe the high availability mechanism implemented in S4, then we describe the checkpointing and recovery mechanism, and how to customize it, then we describe future improvements.
+
+
+# Fail-over mechanism
+
+In order to guarantee availability in the presence of sudden node failures, S4 provides a mechanism to automatically detect failed nodes and redirect messages to a standby node.
+
+
+The following figure illustrates this fail-over mechanism:
+
+
+
+
+This technique provides high availability but does not prevent state loss.
+
+## Configuration
+
+
+##### Number of standby nodes
+
+S4 clusters are defined with a fixed number of tasks (\~ partitions). If you have n partitions and start m nodes, with m>n, you get m-n standby nodes.
+
+##### Failure detection timeout
+
+Zookeeper considers a node is dead when it cannot reach it after a the session timeout. The session timeout is specified by the client upon connection, and is at minimum twice the tickTime (heartbeat) specified in the Zookeeper ensemble configuration.
+
+
+# Checkpointing and recovery
+
+### A closer look at the problem
+
+Upon node crash, the fail-over mechanism brings a new and fresh node to the cluster. When this node is brought into the cluster, it has no state, no instantiated PE. Messages start arriving at this node, and trigger keyed PE instantiations.
+
+
+If there is no checkpointing and recovery mechanism, those PEs start with an empty state.
+
+
+### S4 strategy for solving the problem
+
+For PEs to recover a previous state, the technique we use is to:
+
+
+- *periodically checkpoint* the state of PEs across the S4 cluster
+- *lazily recover* (triggered by messages)
+
+This means that if there is a previous state that was checkpointed, and that a new PE is instantiated because a new key is seen, the PE instance will fetch the corresponding checkpoint, recover the corresponding state, and only then start processing events. State loss is minimal!
+
+
+### Design
+
+##### Checkpointing
+
+In order to minimize the latency, checkpointing is _uncoordinated_ and _asynchronous_.
+Uncoordinated checkpointing means that each checkpoint is taken independently, without aiming at global consistency.
+Asynchronous checkpointing aims at minimizing the impact on the event processing execution path.
+
+
+Taking a checkpoint is a 2 steps operations, both handled outside of the event processing path:
+
+
+- serialize the PE instance
+- save the serialized PE instance to remote storage
+
+
+The following figure shows the various components involved: the checkpointing framework handles the serialization and passes serialized state to a pluggable storage backend:
+
+
+
+
+
+##### Recovery
+
+In order to optimize the usage of resources, recovery is _lazy_, which means it only happens when necessary.
+When a message for a new key arrives in the recovered S4 node, a new PE instance is created, and the system tries to fetch a previous checkpoint from storage. If there is a previous state, it is copied to the newly created PE instance. (This implies deserializing a previous object and copying its fields).
+
+
+### Configuration and customization
+
+##### Requirements
+
+A PE can be checkpointed if:
+
+
+- the PE class provides an empty no-arg constructor (that restriction should be lifted in next releases)
+- it has non transient serializable fields (and by opposition, transient fields will never be checkpointed)
+
+
+##### Checkpointing application configuration
+
+Checkpointing intervals are defined per prototype, in time intervals or event counts (for now). This is specified in the application module, using API methods from the ProcessingElement class, and passing a CheckpointingConfiguration object. Please refer to the API documentation.
+
+
+The twitter example application shipped in the distribution is already configured for enabling checkpointing. See the [TwitterCounterApp](https://git-wip-us.apache.org/repos/asf?p=incubator-s4.git;a=blob;f=test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TwitterCounterApp.java;h=5d7855fa5aee6cbe693fa47c1ebad03da316f42b) class.
+
+
+For instance, here is how to specify a checkpointing frequency of 20s on the TopNTopicPE prototype:
+
+
+ topNTopicPE.setCheckpointingConfig(new CheckpointingConfig.Builder(CheckpointingMode.TIME).frequency(20).timeUnit(TimeUnit.SECONDS).build());
+
+
+##### Enabling checkpointing
+
+This is a node configuration. You need to inject a checkpointing module that speficies a CheckpointingFramework implementation (please use org.apache.s4.core.ft.SafeKeeper) and a backend storage implementation. The backend storage implements the StateStorage interface.
+
+
+We provide a default module (FileSystemBackendCheckpointingModule) that uses a file system backend (DefaultFileSystemStateStorage). It can be used with an NFS setup and introduces no dependency. You may use it by starting an S4 node in the following manner:
+
+
+ ./s4 node -c=cluster1 -emc=org.apache.s4.core.ft.FileSystemBackendCheckpointingModule
+
+
+##### Customizing the checkpointing backend
+
+It is quite straightforward to implement backends for other kinds of storage (key value stores, datagrid, cache, RDBMS). Using an alternative backend is as simple as providing a new module to the S4 node. Here is an example of a module using a 'Cool' backend implementation:
+
+
+ public class CoolBackendCheckpointingModule extends AbstractModule {
+ @Override
+ protected void configure() {
+ bind(StateStorage.class).to(CoolStateStorage.class);
+ bind(CheckpointingFramework.class).to(SafeKeeper.class);
+ }
+ }
+
+
+##### Overriding checkpointing and recovery operations
+
+By default, S4 uses [kryo](http://code.google.com/p/kryo) to serialize and deserialize checkpoints, but it is possible to use a different mechanism, by overriding the `checkpoint()`, `serializeState()` and `restoreState()` methods of the `ProcessingElement` class.
+
+
+PEs are eligible for checkpointing when their state is 'dirty'. The dirty flag is checked through the `isDirty()` method, and cleared by calling the `clearDirty()` method. In some cases, dependent on the application code, only some of the events may actually change the state of the PE. You should override these methods in order to avoid unjustified checkpointing operations.
+
+
+##### Tuning
+
+The checkpointing framework has a number of overridable parameters, mostly for sizing thread pools:
+
+
+* Serialization thread pool
+ * s4.checkpointing.serializationMaxThreads (default = 1)
+ * s4.checkpointing.serializationThreadKeepAliveSeconds (default = 120)
+ * s4.checkpointing.serializationMaxOutstandingRequests (default = 1000)
+
+* Storage backend thread pool
+ * s4.checkpointing.storageMaxThreads (default = 1)
+ * s4.checkpointing.storageThreadKeepAliveSeconds (default = 120)
+ * s4.checkpointing.storageMaxOutstandingRequests (default = 1000)
+
+* Fetching thread pool: fetching is a blocking operation, which can timeout:
+ * s4.checkpointing.fetchingMaxThreads (default = 1)
+ * s4.checkpointing.fetchingThreadKeepAliveSeconds (default = 120)
+ * s4.checkpointing.fetchingMaxWaitMs (default = 1000) (corresponds to the timeout)
+
+* In the case the backend is unresponsive, it can be bypassed:
+ * s4.checkpointing.fetchingMaxConsecutiveFailuresBeforeDisabling (default = 10)
+ * s4.checkpointing.fetchingDisabledDurationMs (default = 600000)
\ No newline at end of file
diff --git a/website/content/doc/0.6.0/index.md b/website/content/doc/0.6.0/index.md
new file mode 100644
index 0000000..521223c
--- /dev/null
+++ b/website/content/doc/0.6.0/index.md
@@ -0,0 +1,30 @@
+---
+title: S4 0.6.0
+---
+
+> This is the documentation for S4 0.6.0. For previous versions, please refer to the [wiki](https://cwiki.apache.org/confluence/display/S4/S4+Wiki)
+
+S4 (Simple Scalable Streaming System) is a general-purpose, distributed, scalable, fault-tolerant, pluggable platform that allows programmers to easily develop applications for processing continuous, unbounded streams of data.
+
+
+
+## Getting Started
+
+* You may start with an [overview](overview) of the platform
+* Then follow a [walkthrough](walkthrough) for an hands-on introduction
+* And [here](dev_tips) are some tips to ease the development process
+
+## Configuration
+
+* How to [customize the platform and pass configuration parameters](configuration)
+* How to [add application dependencies](application_dependencies)
+* How to [dispatch events ](event_dispatch) within an application and between applications
+
+## Features
+
+* Details about [fault tolerance](fault_tolerance)
+
+## Troubleshooting
+
+* Try the [FAQ](https://cwiki.apache.org/confluence/display/S4/FAQ)
+* Try the [mailing lists](https://cwiki.apache.org/S4/s4-apache-mailing-lists.html)
\ No newline at end of file
diff --git a/website/content/doc/0.6.0/overview.md b/website/content/doc/0.6.0/overview.md
new file mode 100644
index 0000000..709795b
--- /dev/null
+++ b/website/content/doc/0.6.0/overview.md
@@ -0,0 +1,106 @@
+---
+title: S4 0.6.0 overview
+---
+
+
+
+# What is S4?
+
+S4 is a general-purpose,near real-time, distributed, decentralized, scalable, event-driven, modular platform that allows programmers to easily implement applications for processing continuous unbounded streams of data.
+
+
+S4 0.5 focused on providing a functional complete refactoring.
+
+S4 0.6 builds on this basis and brings plenty of exciting features, in particular:
+
+* *performance improvements*: stream throughput improved by 1000 % (~200k messages / s / stream)
+* improved [configurability](S4:Configuration - 0.6.0], for both the S4 platform and deployed applications
+* *elasticity* and fine partition tuning, through an integration with Apache Helix
+
+
+# What are the cool features?
+
+**Flexible deployment**:
+
+* By default keys are homogeneously sparsed over the cluster: helps balance the load, especially for fine grained partitioning
+* S4 also provides fine control over the partitioning
+* Features automatic rebalancing
+
+**Modular design**:
+
+* both the platform and the applications are built by dependency injection, and configured through independent modules.
+* makes it easy to customize the system according to specific requirements
+* pluggable event serving policies: load shedding, throttling, blocking
+
+**Dynamic and loose coupling of S4 applications**:
+
+* through a pub-sub mechanism
+* makes it easy to:
+** assemble subsystems into larger systems
+** reuse applications
+** separate pre-processing
+** provision, control and update subsystems independently
+
+
+**[Fault tolerant](fault_tolerance)**
+
+
+* *Fail-over* mechanism for high availability
+* *Checkpointing and recovery* mechanism for minimizing state loss
+
+**Pure Java**: statically typed, easy to understand, to refactor, and to extend
+
+
+
+
+# How does it work?
+
+## Some definitions
+
+**Platform**
+
+* S4 provides a runtime distributed platform that handles communication, scheduling and distribution across containers.
+* Distributed containers are called *S4 nodes*
+* S4 nodes are deployed on *S4 clusters*
+* S4 clusters define named ensembles of S4 nodes, with a fixed size
+* The size of an S4 cluster corresponds to the number of logical *partitions* (sometimes referred to as _tasks_)
+
+**Applications**
+
+
+
+* Users develop applications and deploy them on S4 clusters
+* Applications are built from:
+** *Processing elements* (PEs)
+** *Streams* that interconnect PEs
+
+* PEs communicate asynchronously by sending *events* on streams.
+* Events are dispatched to nodes according to their key
+
+**External streams** are a special kind of stream that:
+
+
+
+* send events outside of the application
+* receive events from external sources
+* can interconnect and assemble applications into larger systems.
+
+**Adapters** are S4 applications that can convert external streams into streams of S4 events. Since adapters are also S4 applications, they can be scaled easily.
+
+
+
+
+## A hierarchical perspective on S4
+
+The following diagram sums-up the key concepts in a hierarchical fashion:
+
+
+
+# Where can I find more information?
+
+* [The website](http://incubator.apache.org/s4/) is a good starting point.
+* [The wiki](https://cwiki.apache.org/confluence/display/S4/) currently contains the most up-to-date information: general information (this page), configuration, examples.
+* Questions can be asked through the [mailing lists](https://cwiki.apache.org/confluence/display/S4/S4+Apache+mailing+lists)
+* The source code is available throught [git](https://git-wip-us.apache.org/repos/asf?p=incubator-s4.git], [here](http://incubator.apache.org/s4/contrib/) are instructions for fetching the code.
+* A nice set of [slides](http://www.slideshare.net/leoneu/20111104-s4-overview) was used for a presentation at Stanford in November 2011.
+* The driving ideas are detailed in a [conference publication](http://www.4lunas.org/pub/2010-s4.pdf) from KDCloud'11 (joint workshop with ICDM'11)
\ No newline at end of file
diff --git a/website/content/doc/0.6.0/walkthrough.md b/website/content/doc/0.6.0/walkthrough.md
new file mode 100644
index 0000000..f819331
--- /dev/null
+++ b/website/content/doc/0.6.0/walkthrough.md
@@ -0,0 +1,304 @@
+---
+title: Walkthrough
+---
+
+
+> Improvements from S4 0.5.0 include a more convenient configuration system, illustrated here: all platform and application parameters are specified when configuring/deploying the app.
+
+
+# Install S4
+
+There are 2 ways:
+
+* [Download](http://incubator.apache.org/s4/download/) the 0.6.0 release
+
+> We recommend getting the "source" release and building it
+
+* or checkout from the Apache git repository, by following the [instructions](/contrib). The 0.6.0 tag corresponds to the current release.
+
+If you get the binary release, s4 scripts are immediately available. Otherwise you must build the project:
+
+* Compile and install S4 in the local maven repository: (you can also let the tests run without the -DskipTests option)
+
+ S4:incubator-s4$ ./gradlew install -DskipTests
+ .... verbose logs ...
+
+* Build the startup scripts:
+
+ S4:incubator-s4$ ./gradlew s4-tools:installApp
+ .... verbose logs
+ ...:s4-tools:installApp
+
+
+----
+
+
+# Start a new application
+
+S4 provides some scripts in order to simplify development and testing of applications. Let's see how to create a new project and start a sample application.
+
+## Create a new project
+
+* Create a new application template (here, we create it in the /tmp directory):
+
+ S4:incubator-s4$ ./s4 newApp myApp -parentDir=/tmp
+ ... some instructions on how to start ...
+
+* This creates a sample application in the specified directory, with the following structure:
+
+ build.gradle --> the template build file, that you'll need to customize
+ gradlew --> references the gradlew script from the S4 installation
+ s4 --> references the s4 script from the S4 installation, and adds an "adapter" task
+ src/ --> sources (maven-like structure)
+
+
+## Have a look at the sample project content
+
+The src/main/java/hello directory contains 3 files:
+
+* HelloPE.java : a very simple PE that simply prints the name contained in incoming events
+ // ProcessingElement provides integration with the S4 platform
+ public class HelloPE extends ProcessingElement {
+
+ // you should define downstream streams here and inject them in the app definition
+
+ // PEs can maintain some state
+ boolean seen = false;
+
+ // This method is called upon a new Event on an incoming stream.
+ // You may overload it for handling instances of your own specialized subclasses of Event
+ public void onEvent(Event event) {
+ System.out.println("Hello " + (seen ? "again " : "") + event.get("name") + "!");
+ seen = true;
+ }
+ // skipped remaining methods
+
+* HelloApp.java: defines a simple application: exposes an input stream ("names"), connected to the HelloPE. See [the event dispatch configuration page](event_dispatch) for more information about how events are dispatched.
+ // App parent class provides integration with the S4 platform
+ public class HelloApp extends App {
+
+ @Override
+ protected void onStart() {
+ }
+
+ @Override
+ protected void onInit() {
+ // That's where we define PEs and streams
+ // create a prototype
+ HelloPE helloPE = createPE(HelloPE.class);
+ // Create a stream that listens to the "lines" stream and passes events to the helloPE instance.
+ createInputStream("names", new KeyFinder<Event>() {
+ // the KeyFinder is used to identify keys
+ @Override
+ public List<String> get(Event event) {
+ return Arrays.asList(new String[] { event.get("name") });
+ }
+ }, helloPE);
+ }
+ // skipped remaining methods
+
+* HelloInputAdapter is a simple adapter that reads character lines from a socket, converts them into events, and sends the events to interested S4 apps, through the "names" stream
+
+## Run the sample app
+
+In order to run an S4 application, you need :
+
+* to set-up a cluster: provision a cluster and start S4 nodes for that cluster
+* to package the app
+* to publish the app on the cluster
+
+# Set-up the cluster:
+
+* In 2 steps:
+
+ 1. Start a Zookeeper server instance (-clean option removes previous ZooKeeper data, if any):
+
+ S4:incubator-s4$ ./s4 zkServer - clean
+ S4:myApp$ calling referenced s4 script : /Users/S4/tmp/incubator-s4/s4
+ [main] INFO org.apache.s4.tools.ZKServer - Starting zookeeper server on port [2181]
+ [main] INFO org.apache.s4.tools.ZKServer - cleaning existing data in [/var/folders/8V/8VdgKWU3HCiy2yV4dzFpDk+++TI/-Tmp-/tmp/zookeeper/data] and [/var/folders/8V/8VdgKWU3HCiy2yV4dzFpDk+++TI/-Tmp-/tmp/zookeeper/log]
+
+ 1. Define a new cluster. Say a cluster named "cluster1" with 2 partitions, nodes listening to ports starting from 12000:
+
+ S4:myApp$ ./s4 newCluster -c=cluster1 -nbTasks=2 -flp=12000
+ calling referenced s4 script : /Users/S4/tmp/incubator-s4/s4
+ [main] INFO org.apache.s4.tools.DefineCluster - preparing new cluster [cluster1] with [2] node(s)
+ [main] INFO org.apache.s4.tools.DefineCluster - New cluster configuration uploaded into zookeeper
+
+* Alternatively you may combine these two steps into a single one, by passing the cluster configuration inline with the `zkServer` command:
+
+ S4:incubator-s4$ ./s4 zkServer -clusters=c=cluster1:flp=12000:nbTasks=2 -clean
+
+* Start 2 S4 nodes with the default configuration, and attach them to cluster "cluster1" :
+
+ S4:myApp$ ./s4 node -c=cluster1
+ calling referenced s4 script : /Users/S4/tmp/incubator-s4/s4
+ 15:50:18.996 [main] INFO org.apache.s4.core.Main - Initializing S4 node with :
+ - comm module class [org.apache.s4.comm.DefaultCommModule]
+ - comm configuration file [default.s4.comm.properties from classpath]
+ - core module class [org.apache.s4.core.DefaultCoreModule]
+ - core configuration file[default.s4.core.properties from classpath]
+ -extra modules: []
+ [main] INFO org.apache.s4.core.Main - Starting S4 node. This node will automatically download applications published for the cluster it belongs to
+and again (maybe in another shell):
+
+ S4:myApp$ ./s4 node -c=cluster1
+
+* Build, package and publish the app to cluster1:
+ * This is done in 2 separate steps:
+ 1. Create an s4r archive. The following creates an archive named myApp.s4r (here you may specify an arbitrary name) in build/libs.
+Again specifying the app class is optional :
+
+ ./s4 s4r -a=hello.HelloApp -b=`pwd`/build.gradle myApp
+
+ 1. Publish the s4r archive (you may first copy it to a more adequate place). The name of the app is arbitrary:
+
+ ./s4 deploy -s4r=`pwd`/build/libs/myApp.s4r -c=cluster1 -appName=myApp
+
+* S4 nodes will detect the new application, download it, load it and start it. You will get something like:
+
+ [ZkClient-EventThread-15-localhost:2181] INFO o.a.s.d.DistributedDeploymentManager - Detected new application(s) to deploy {}[myApp]
+ [ZkClient-EventThread-15-localhost:2181] INFO org.apache.s4.core.Server - Local app deployment: using s4r file name [myApp] as application name
+ [ZkClient-EventThread-15-localhost:2181] INFO org.apache.s4.core.Server - App class name is: hello.HelloApp
+ [ZkClient-EventThread-15-localhost:2181] INFO o.a.s4.comm.topology.ClusterFromZK - Changing cluster topology to { nbNodes=0,name=unknown,mode=unicast,type=,nodes=[]} from null
+ [ZkClient-EventThread-15-localhost:2181] INFO o.a.s4.comm.topology.ClusterFromZK - Adding topology change listener:org.apache.s4.comm.tcp.TCPEmitter@79b2591c
+ [ZkClient-EventThread-15-localhost:2181] INFO o.a.s.comm.topology.AssignmentFromZK - New session:87684175268872203; state is : SyncConnected
+ [ZkClient-EventThread-19-localhost:2181] INFO o.a.s4.comm.topology.ClusterFromZK - Changing cluster topology to { nbNodes=1,name=cluster1,mode=unicast,type=,nodes=[{partition=0,port=12000,machineName=myMachine.myNetwork,taskId=Task-0}]} from { nbNodes=0,name=unknown,mode=unicast,type=,nodes=[]}
+ [ZkClient-EventThread-15-localhost:2181] INFO o.a.s.comm.topology.AssignmentFromZK - Successfully acquired task:Task-1 by myMachine.myNetwork
+ [ZkClient-EventThread-19-localhost:2181] INFO o.a.s4.comm.topology.ClusterFromZK - Changing cluster topology to { nbNodes=2,name=cluster1,mode=unicast,type=,nodes=[{partition=0,port=12000,machineName=myMachine.myNetwork,taskId=Task-0}, {partition=1,port=12001,machineName=myMachine.myNetwork,taskId=Task-1}]} from { nbNodes=1,name=cluster1,mode=unicast,type=,nodes=[{partition=0,port=12000,machineName=myMachine.myNetwork,taskId=Task-0}]}
+ [ZkClient-EventThread-15-localhost:2181] INFO o.a.s4.comm.topology.ClustersFromZK - New session:87684175268872205
+ [ZkClient-EventThread-15-localhost:2181] INFO o.a.s4.comm.topology.ClustersFromZK - Detected new stream [names]
+ [ZkClient-EventThread-15-localhost:2181] INFO o.a.s4.comm.topology.ClustersFromZK - New session:87684175268872206
+ [ZkClient-EventThread-15-localhost:2181] INFO o.a.s4.comm.topology.ClusterFromZK - Changing cluster topology to { nbNodes=2,name=cluster1,mode=unicast,type=,nodes=[{partition=0,port=12000,machineName=myMachine.myNetwork,taskId=Task-0}, {partition=1,port=12001,machineName=myMachine.myNetwork,taskId=Task-1}]} from null
+ [ZkClient-EventThread-15-localhost:2181] INFO org.apache.s4.core.Server - Loaded application from file /tmp/deploy-test/cluster1/myApp.s4r
+ [ZkClient-EventThread-15-localhost:2181] INFO o.a.s.d.DistributedDeploymentManager - Successfully installed application myApp
+ [ZkClient-EventThread-15-localhost:2181] DEBUG o.a.s.c.g.OverloadDispatcherGenerator - Dumping generated overload dispatcher class for PE of class [class hello.HelloPE]
+ [ZkClient-EventThread-15-localhost:2181] DEBUG o.a.s4.comm.topology.ClustersFromZK - Adding input stream [names] for app [-1] in cluster [cluster1]
+ [ZkClient-EventThread-15-localhost:2181] INFO org.apache.s4.core.App - Init prototype [hello.HelloPE].
+
+
+Great! The application is now deployed on 2 S4 nodes.
+
+You can check the status of the application, nodes and streams with the "status" command:
+
+ ./s4 status
+
+Now what we need is some input!
+
+We can get input through an adapter, i.e. an S4 app that converts an external stream into S4 events, and injects the events into S4 clusters. In the sample application, the adapter is a very basic class, that extends App, listens to an input socket on port 15000, and converts each received line of characters into a generic S4 event, in which the line data is kept in a "name" field. We specify :
+
+* the adapter class
+* the name of the output stream
+* the cluster where to deploy this app
+
+For easy testing, we provide a facility to start a node with an adapter app without having to package the adapter app.
+
+* First, we need to define a new S4 subcluster for that app:
+
+ S4:myApp$ ./s4 newCluster -c=cluster2 -nbTasks=1 -flp=13000
+
+* Then we configure the application:
+ * we specify the adapter class (app class)
+ * we use "names" for identifying the output stream (this is the same name used as input by the myApp app)
+ * _there is also a -s4r parameter, indicating where to fetch the application package from. We don't need it here, since we skip that step and use a special "adapter" tool_
+
+ ./s4 deploy -appClass=hello.HelloInputAdapter -p=s4.adapter.output.stream=names -c=cluster2 -appName=adapter
+
+* Then we simply start the adapter (there is no packaging and copying of the S4R package)
+> The adapter command must be run from the root of your S4 project (myApp dir in our case).
+
+ ./s4 adapter -c=cluster2
+
+* Now let's just provide some data to the external stream (our adapter is listening to port 15000):
+
+ S4:~$ echo "Bob" | nc localhost 15000
+
+* One of the nodes should output in its console:
+
+ Hello Bob!
+
+
+> If you keep sending messages, nodes will alternatively display the "hello" messages because the adapter app sends keyless events on the "names" stream in a round-robin fashion by default.
+
+## What happened?
+
+The following figures illustrate the various steps we have taken. The local file system is used as the S4 application repository in our example.
+
+
+
+
+----
+
+
+# Run the Twitter trending example
+
+Let's have a look at another application, that computes trendy Twitter topics by listening to the spritzer stream from the Twitter API. This application was adapted from a previous example in S4 0.3.
+
+## Overview
+
+This application is divided into:
+
+* twitter-counter , in test-apps/twitter-counter/ : extracts topics from tweets and maintains a count of the most popular ones, periodically dumped to disk
+* twitter-adapter, in test-apps/twitter-adapter/ : listens to the feed from Twitter, converts status text into S4 events, and passes them to the "RawStatus" stream
+
+Have a look at the code in these directories. You'll note that:
+
+* the build.gradle file must be tailored to include new dependencies (twitter4j libs in twitter-adapter)
+* events are partitioned through various keys
+
+## Run it!
+
+> Note: You need a twitter4j.properties file in your home directory with the following content (debug is optional):
+
+ debug=true
+ user=<a twitter username>
+ password=<matching password>
+
+* Start a Zookeeper instance. From the S4 base directory, do:
+
+ ./s4 zkServer
+
+* Define 2 clusters : 1 for deploying the twitter-counter app, and 1 for the adapter app
+
+ ./s4 newCluster -c=cluster1 -nbTasks=2 -flp=12000; ./s4 newCluster -c=cluster2 -nbTasks=1 -flp=13000
+
+* Start 2 app nodes (you may want to start each node in a separate console) :
+
+ ./s4 node -c=cluster1
+ ./s4 node -c=cluster1
+
+* Start 1 node for the adapter app:
+
+ ./s4 node -c=cluster2 -p=s4.adapter.output.stream=RawStatus
+
+* Deploy twitter-counter app (you may also first build the s4r then publish it, as described in the previous section)
+
+ ./s4 deploy -appName=twitter-counter -c=cluster1 -b=`pwd`/test-apps/twitter-counter/build.gradle
+
+* Deploy twitter-adapter app. In this example, we don't directly specify the app class of the adapter, we use the deployment approach for apps (remember, the adapter is also an app).
+
+ ./s4 deploy -appName=twitter-adapter -c=cluster2 -b=`pwd`/test-apps/twitter-adapter/build.gradle
+
+* Observe the current 10 most popular topics in file TopNTopics.txt. The file gets updated at regular intervals, and only outputs topics with a minimum of 10 occurrences, so you may have to wait a little before the file is updated :
+
+ tail -f TopNTopics.txt
+
+* You may also check the status of the S4 node with:
+
+ ./s4 status
+
+----
+
+# What next?
+
+You have now seen some basics applications, and you know how to run them, and how to get events into the system. You may now try to code your own apps with your own data.
+
+[This page](../application_dependencies) will help for specifying your own dependencies.
+
+There are more parameters available for the scripts (typing the name of the task will list the options). In particular, if you want distributed deployments, you'll need to pass the Zookeeper connection strings when you start the nodes.
+
+You may also customize the communication and the core layers of S4 by tweaking configuration files and modules.
+
+Last, the [javadoc](http://people.apache.org/~mmorel/apache-s4-0.6.0-incubating-doc/javadoc/) will help you when writing applications.
+
+We hope this will help you start rapidly, and remember: we're happy to help!
\ No newline at end of file
diff --git a/website/content/download.haml b/website/content/download.haml
new file mode 100644
index 0000000..ec45db4
--- /dev/null
+++ b/website/content/download.haml
@@ -0,0 +1,23 @@
+---
+title: Download
+---
+#main{:class=>"span-18 colborder"}
+
+ %h1{:class=>"titleheader"}= "Downloading S4"
+
+ %p= "The current stable version is 0.5.0-incubating. You can verify your download by following <a href='http://www.apache.org/info/verification.html'>these procedures</a> and using <a href='http://www.apache.org/dist/incubator/s4/KEYS'>these keys</a>."
+
+ / 0.5.0
+ %h2{:class=>"secheader"}= "0.5.0 release"
+
+ %ul
+ %li <a href='http://www.apache.org/dist/incubator/s4/s4-0.5.0-incubating/RELEASE_NOTES.html'>Release notes</a>
+ %li Source release: <a href='http://www.apache.org/dist/incubator/s4/s4-0.5.0-incubating/apache-s4-0.5.0-incubating-src.zip'>s4-0.5.0-incubating-src.zip</a> (<a href='http://www.apache.org/dist/incubator/s4/s4-0.5.0-incubating/apache-s4-0.5.0-incubating-src.zip.asc'>asc</a>, <a href='http://www.apache.org/dist/incubator/s4/s4-0.5.0-incubating/apache-s4-0.5.0-incubating-src.zip.md5'>md5</a>)
+ %li Binary release: <a href='http://www.apache.org/dist/incubator/s4/s4-0.5.0-incubating/apache-s4-0.5.0-incubating-bin.zip'>s4-0.5.0-incubating-bin.zip</a> (<a href='http://www.apache.org/dist/incubator/s4/s4-0.5.0-incubating/apache-s4-0.5.0-incubating-bin.zip.asc'>asc</a>, <a href='http://www.apache.org/dist/incubator/s4/s4-0.5.0-incubating/apache-s4-0.5.0-incubating-bin.zip.md5'>md5</a>)
+
+ / older-releases
+ %h2{:class=>"secheader"}= "Older version"
+
+ %p="There is no 0.4.x release"
+
+ %p="Older version can be downloaded through github. See instructions <a href='http://docs.s4.io/tutorials/getting_started.html'>here</a>."
diff --git a/website/content/images/body.gif b/website/content/images/body.gif
new file mode 100644
index 0000000..6e1250a
--- /dev/null
+++ b/website/content/images/body.gif
Binary files differ
diff --git a/website/content/images/doc/0.6.0/S4_hierarchical_archi.png b/website/content/images/doc/0.6.0/S4_hierarchical_archi.png
new file mode 100644
index 0000000..14d7d4e
--- /dev/null
+++ b/website/content/images/doc/0.6.0/S4_hierarchical_archi.png
Binary files differ
diff --git a/website/content/images/doc/0.6.0/checkpointing-framework.png b/website/content/images/doc/0.6.0/checkpointing-framework.png
new file mode 100644
index 0000000..912ffe4
--- /dev/null
+++ b/website/content/images/doc/0.6.0/checkpointing-framework.png
Binary files differ
diff --git a/website/content/images/doc/0.6.0/failover.png b/website/content/images/doc/0.6.0/failover.png
new file mode 100644
index 0000000..3fc4376
--- /dev/null
+++ b/website/content/images/doc/0.6.0/failover.png
Binary files differ
diff --git a/website/content/images/doc/0.6.0/s4_node_layers.png b/website/content/images/doc/0.6.0/s4_node_layers.png
new file mode 100644
index 0000000..96a82d0
--- /dev/null
+++ b/website/content/images/doc/0.6.0/s4_node_layers.png
Binary files differ
diff --git a/website/content/images/doc/0.6.0/sampleAppDeployment.png b/website/content/images/doc/0.6.0/sampleAppDeployment.png
new file mode 100644
index 0000000..eea4b25
--- /dev/null
+++ b/website/content/images/doc/0.6.0/sampleAppDeployment.png
Binary files differ
diff --git a/website/content/images/doc/0.6.0/sources/s4_node_layers.odg b/website/content/images/doc/0.6.0/sources/s4_node_layers.odg
new file mode 100644
index 0000000..10315e9
--- /dev/null
+++ b/website/content/images/doc/0.6.0/sources/s4_node_layers.odg
Binary files differ
diff --git a/website/content/images/gradient.png b/website/content/images/gradient.png
new file mode 100644
index 0000000..7a75acb
--- /dev/null
+++ b/website/content/images/gradient.png
Binary files differ
diff --git a/website/content/images/grid.png b/website/content/images/grid.png
new file mode 100644
index 0000000..129d4a2
--- /dev/null
+++ b/website/content/images/grid.png
Binary files differ
diff --git a/website/content/images/header.png b/website/content/images/header.png
new file mode 100644
index 0000000..7896bee
--- /dev/null
+++ b/website/content/images/header.png
Binary files differ
diff --git a/website/content/images/just_s4.png b/website/content/images/just_s4.png
new file mode 100644
index 0000000..bf7e355
--- /dev/null
+++ b/website/content/images/just_s4.png
Binary files differ
diff --git a/website/content/images/mp.png b/website/content/images/mp.png
new file mode 100644
index 0000000..fb02ab5
--- /dev/null
+++ b/website/content/images/mp.png
Binary files differ
diff --git a/website/content/images/nav_bg.png b/website/content/images/nav_bg.png
new file mode 100644
index 0000000..ba1f78f
--- /dev/null
+++ b/website/content/images/nav_bg.png
Binary files differ
diff --git a/website/content/images/osi.png b/website/content/images/osi.png
new file mode 100644
index 0000000..0ccebfe
--- /dev/null
+++ b/website/content/images/osi.png
Binary files differ
diff --git a/website/content/images/s4.png b/website/content/images/s4.png
new file mode 100644
index 0000000..7b86c2f
--- /dev/null
+++ b/website/content/images/s4.png
Binary files differ
diff --git a/website/content/images/s4_test.png b/website/content/images/s4_test.png
new file mode 100644
index 0000000..4744911
--- /dev/null
+++ b/website/content/images/s4_test.png
Binary files differ
diff --git a/website/content/images/sec_os.png b/website/content/images/sec_os.png
new file mode 100644
index 0000000..2873bcc
--- /dev/null
+++ b/website/content/images/sec_os.png
Binary files differ
diff --git a/website/content/index.haml b/website/content/index.haml
new file mode 100644
index 0000000..3962218
--- /dev/null
+++ b/website/content/index.haml
@@ -0,0 +1,78 @@
+---
+title: Distributed Stream Computing Platform
+---
+#main{:class=>"span-18 colborder"}
+ #about{:class=>"span-18 last"}
+
+ #notice{:class=>"span-17 last"}
+ #notice_text= "August 2012: <b>S4 0.5.0 has been released!</b> Get it <a href='/download'>here</a>!."
+
+ #notice{:class=>"span-17 last"}
+ #notice_text= "S4 0.5.0 \"Piper\" is a complete refactoring of the previous version of S4. It provides a clearer API and a more robust implementation. It features TCP based communications, state recovery, and a new set of tools. More information is available in an <a href='https://cwiki.apache.org/confluence/display/S4/S4+0.5.0+Overview'>overview</a> and you may also check the <a href='http://www.apache.org/dist/incubator/s4/s4-0.5.0-incubating/RELEASE_NOTES.html'>release notes</a>."
+
+ %p{:class=>'large'}= "S4 is a <span class='high'>general-purpose</span>, <span class='high'>distributed</span>, <span class='high'>scalable</span>, <span class='high'>fault-tolerant</span>, <span class='high'>pluggable</span> platform that allows programmers to easily develop applications for processing continuous unbounded streams of data."
+
+ .horrule
+
+ #sec_motivation{:class=>"span-9"}
+ %h2{:class=>"secheader"}= "motivation"
+ %p= "S4 fills the gap between complex proprietary systems and batch-oriented open source computing platforms. We aim to develop a high performance computing platform that hides the complexity inherent in parallel processing system from the application programmer."
+ #sec_implementation{:class=>"span-9 last"}
+ %h2{:class=>"secheader"}= "implementation"
+ %p= "The core platform is written in <span class='high'>Java</span>. The implementation is modular and pluggable, and S4 applications can be easily and dynamically combined for creating more sophisticated stream processing systems."
+
+ .horrule
+
+ #sec_osi{:class=>"span-18 last"}
+ %h2{:class=>"secheader"}= "open source"
+ #os_logo{:class=>"span-2"}
+ %img{:src=>"http://incubator.apache.org/images/apache-incubator-logo.png"}
+ #os_desc{:class=>"span-14 last"}
+ %p{:style=>"padding-top:10px; padding-left:170px;"}= "S4 was initially released by Yahoo! Inc. in October 2010 and is an Apache Incubator project since September 2011. It is licensed under the Apache 2.0 license."
+
+ .horrule
+
+ #sec_overview{:class=>"span-18 last"}
+
+ %h2{:class=>"secheader"}= "overview"
+ #ov_1{:class=>"span-6"}
+ %h4{:class=>"subsecheader"}= "proven"
+ %p= "S4 has been deployed in production systems at Yahoo! to process thousands of search queries per second."
+ #ov_2{:class=>"span-6"}
+ %h4{:class=>"subsecheader"}= "decentralized"
+ %p= "All nodes are symmetric with no centralized service and no single point of failure. This greatly simplifies deployments and cluster configuration changes."
+ #ov_3{:class=>"span-6 last"}
+ %h4{:class=>"subsecheader"}= "scalable"
+ %p= "Throughput increases linearly as additional nodes are added to the cluster. There is no predefined limit on the number of nodes that can be supported."
+
+ #ov_4{:class=>"span-6"}
+ %h4{:class=>"subsecheader"}= "extensible"
+ %p= "Applications can easily be written and deployed using a simple API. Many basic applications for stream processing are available out of the box and more are being written."
+ #ov_5{:class=>"span-6"}
+ %h4{:class=>"subsecheader"}= "cluster management"
+ %p= "S4 hides all cluster management tasks using a communication layer built on top of <a href='http://hadoop.apache.org/zookeeper'>ZooKeeper</a>, a distributed, open-source coordination service for distributed applications."
+ #ov_6{:class=>"span-6 last"}
+ %h4{:class=>"subsecheader"}= "fault-tolerance"
+ %p= "When a server in the cluster fails, a stand-by server is automatically activated to take over the tasks. Checkpointing and recovery minimize state loss."
+
+#sidebar{:class=>"span-5 last"}
+ #download
+ #latest_head
+ %span= "S4 Piper release"
+ %br
+ %span{:style=>"font-size: small;"}= "v0.5.0"
+ #latest_link
+ %a{:href=>"https://cwiki.apache.org/confluence/display/S4/S4+piper+walkthrough", :style=>"border: none;"}= "Get Started"
+ #disclaimer
+ %p= "This is an alpha version. There's no guarantee of backwards-compatibility until the 1.0 release."
+ #participate
+ %h2{:class=>"secheader"}= "participate"
+ %ul{:class=>'large greybar'}
+ %li= "<a href='/contrib'>Contribute</a>"
+ %li= "Join the <a href='https://cwiki.apache.org/confluence/display/S4/S4+Apache+mailing+lists' onClick=\"#{google_analytics_event("External", "Forum", "Index - https://cwiki.apache.org/confluence/display/S4/S4+Apache+mailing+lists")}\">forum</a>"
+
+ %h2{:class=>"secheader"}= "recent tweets"
+ #twitter_widget= twitter_widget()
+/ #twitter_follow{:style=>"position: absolute; bottom: 0px;"}
+/ %a{:href=>"http://www.twitter.com/s4project", :onClick=> google_analytics_event("External", "Twitter", "Index - http://www.twitter.com/s4project")}
+/ %img{:src=>"http://twitter-badges.s3.amazonaws.com/follow_bird_us-c.png", :alt=>"Follow s4project on Twitter"}
diff --git a/website/content/style/ie.scss b/website/content/style/ie.scss
new file mode 100644
index 0000000..c2e4489
--- /dev/null
+++ b/website/content/style/ie.scss
@@ -0,0 +1,4 @@
+@import "blueprint";
+
+// Generate the blueprint IE-specific customizations:
+@include blueprint-ie;
diff --git a/website/content/style/nav.scss b/website/content/style/nav.scss
new file mode 100644
index 0000000..f8dbd4f
--- /dev/null
+++ b/website/content/style/nav.scss
@@ -0,0 +1,154 @@
+$border_radius_big: 3px;
+$border_radius_small: 3px;
+
+#navbar {
+ font: Arial,Helvetica,sans-serif;
+ position: absolute;
+ right: 0;
+ top: 35px;
+}
+
+#nav {
+ margin: 0;
+ padding: 7px 6px 0;
+ line-height: 100%;
+
+ border-radius: $border_radius_small;
+ -webkit-border-radius: $border_radius_small;
+ -moz-border-radius: $border_radius_small;
+
+ -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, .4);
+ -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, .4);
+
+ background: #8b8b8b; /* for non-css3 browsers */
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#a9a9a9', endColorstr='#7a7a7a'); /* for IE */
+ background: -webkit-gradient(linear, left top, left bottom, from(#a9a9a9), to(#7a7a7a)); /* for webkit browsers */
+ background: -moz-linear-gradient(top, #a9a9a9, #7a7a7a); /* for firefox 3.6+ */
+
+ border: solid 1px #6d6d6d;
+
+ li {
+ margin: 0 4px;
+ padding: 0 0 6px;
+ float: left;
+ position: relative;
+ list-style: none;
+ }
+
+ a {
+ border: none;
+ font-weight: bold;
+ color: #e7e5e5;
+ text-decoration: none;
+ display: block;
+ padding: 4px 10px;
+ margin: 0;
+ -webkit-border-radius: $border_radius_big;
+ -moz-border-radius: $border_radius_big;
+ text-shadow: 0 1px 1px rgba(0, 0, 0, .3);
+ }
+}
+
+#nav .current a, #nav li:hover > a {
+ background: #d1d1d1; /* for non-css3 browsers */
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ebebeb', endColorstr='#a1a1a1'); /* for IE */
+ background: -webkit-gradient(linear, left top, left bottom, from(#ebebeb), to(#a1a1a1)); /* for webkit browsers */
+ background: -moz-linear-gradient(top, #ebebeb, #a1a1a1); /* for firefox 3.6+ */
+
+ color: #444;
+ //border-top: solid 1px #f8f8f8;
+ -webkit-box-shadow: 0 1px 1px rgba(0, 0, 0, .2);
+ -moz-box-shadow: 0 1px 1px rgba(0, 0, 0, .2);
+ box-shadow: 0 1px 1px rgba(0, 0, 0, .2);
+ text-shadow: 0 1px 0 rgba(255, 255, 255, .8);
+}
+/* sub levels link hover */
+#nav ul li:hover a, #nav li:hover li a {
+ background: none;
+ border: none;
+ color: #666;
+ -webkit-box-shadow: none;
+ -moz-box-shadow: none;
+}
+#nav ul a:hover {
+ background: #0399d4 !important; /* for non-css3 browsers */
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#04acec', endColorstr='#0186ba'); /* for IE */
+ background: -webkit-gradient(linear, left top, left bottom, from(#04acec), to(#0186ba)) !important; /* for webkit browsers */
+ background: -moz-linear-gradient(top, #04acec, #0186ba) !important; /* for firefox 3.6+ */
+
+ color: #fff !important;
+ -webkit-border-radius: 0;
+ -moz-border-radius: 0;
+ text-shadow: 0 1px 1px rgba(0, 0, 0, .1);
+}
+/* level 2 list */
+#nav ul {
+ background: #ddd; /* for non-css3 browsers */
+ filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffff', endColorstr='#cfcfcf'); /* for IE */
+ background: -webkit-gradient(linear, left top, left bottom, from(#fff), to(#cfcfcf)); /* for webkit browsers */
+ background: -moz-linear-gradient(top, #fff, #cfcfcf); /* for firefox 3.6+ */
+
+ display: none;
+ margin: 0;
+ padding: 0;
+ width: 185px;
+ position: absolute;
+ top: 35px;
+ left: 0;
+ border: solid 1px #b4b4b4;
+ -webkit-border-radius: $border_radius_big;
+ -moz-border-radius: $border_radius_big;
+ border-radius: $border_radius_big;
+ -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, .3);
+ -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, .3);
+ box-shadow: 0 1px 3px rgba(0, 0, 0, .3);
+}
+/* dropdown */
+#nav li:hover > ul {
+ display: block;
+}
+#nav ul li {
+ float: none;
+ margin: 0;
+ padding: 0;
+}
+#nav ul a {
+ font-weight: normal;
+ text-shadow: 0 1px 1px rgba(255, 255, 255, .9);
+}
+/* level 3+ list */
+#nav ul ul {
+ left: 181px;
+ top: -3px;
+}
+/* rounded corners for first and last child */
+#nav ul li:first-child > a {
+ -webkit-border-top-left-radius: 9px;
+ -moz-border-radius-topleft: 9px;
+ -webkit-border-top-right-radius: 9px;
+ -moz-border-radius-topright: 9px;
+}
+#nav ul li:last-child > a {
+ -webkit-border-bottom-left-radius: 9px;
+ -moz-border-radius-bottomleft: 9px;
+ -webkit-border-bottom-right-radius: 9px;
+ -moz-border-radius-bottomright: 9px;
+}
+/* clearfix */
+#nav:after {
+ content: ".";
+ display: block;
+ clear: both;
+ visibility: hidden;
+ line-height: 0;
+ height: 0;
+}
+#nav {
+ display: inline-block;
+}
+html[xmlns] #nav {
+ display: block;
+}
+* html #nav {
+ height: 1%;
+}
\ No newline at end of file
diff --git a/website/content/style/partials/_base.scss b/website/content/style/partials/_base.scss
new file mode 100644
index 0000000..f7012ea
--- /dev/null
+++ b/website/content/style/partials/_base.scss
@@ -0,0 +1,11 @@
+// Here is where you can define your constants for your application and to configure the blueprint framework.
+// Feel free to delete these if you want keep the defaults:
+
+$blueprint-grid-columns: 24;
+$blueprint-grid-width: 30px;
+$blueprint-grid-margin: 10px;
+
+
+// If you change your grid column dimensions
+// you can make a new grid background image from the command line like this:
+// compass grid-img 30+10x16
diff --git a/website/content/style/print.scss b/website/content/style/print.scss
new file mode 100644
index 0000000..08f3cfe
--- /dev/null
+++ b/website/content/style/print.scss
@@ -0,0 +1,4 @@
+@import "blueprint";
+
+// Generate the blueprint print styles:
+@include blueprint-print;
diff --git a/website/content/style/screen.scss b/website/content/style/screen.scss
new file mode 100644
index 0000000..ac61e0c
--- /dev/null
+++ b/website/content/style/screen.scss
@@ -0,0 +1,17 @@
+// This import applies a global reset to any page that imports this stylesheet.
+@import "blueprint/reset";
+
+// To configure blueprint, edit the partials/_base.sass file.
+@import "partials/base";
+
+// Import all the default blueprint modules so that we can access their mixins.
+@import "blueprint";
+
+// Import the non-default scaffolding module.
+@import "blueprint/scaffolding";
+
+
+// Generate the blueprint framework according to your configuration:
+@include blueprint;
+
+@include blueprint-scaffolding;
diff --git a/website/content/style/style.scss b/website/content/style/style.scss
new file mode 100644
index 0000000..19a18fb
--- /dev/null
+++ b/website/content/style/style.scss
@@ -0,0 +1,285 @@
+/* globals */
+* { padding: 0; margin: 0; }
+
+$bg_body: #efefef;
+$bg_content: #f4f5f5;
+$sec_header: #7E2217;
+$grad_start: #fafafa;
+$grad_end: #dfdfdf;
+$dark_bg: #dfdfdf;
+$font-color: #7E2217;
+
+body {
+ font-family: "Lucida Grande";
+ font-size: 90%;
+ background-color: $bg_body;
+ color: #333;
+ pre {
+ background-color: $dark_bg;
+ overflow: auto;
+ }
+}
+
+a {
+ color: #000;
+ text-decoration: none;
+ border-bottom: 1px dotted #222;
+}
+
+a:hover { background: #ccc; color: #000;}
+a:visited { background: #ccc; color: #000;}
+
+#wrapper {
+ // background: url("/images/gradient.png") repeat-x scroll left top $bg_content;
+ border-top: 1px solid #bbb;
+ border-bottom: 1px solid #bbb;
+ background: #fff;
+ padding-bottom: 3em;
+ padding-top: 3em;
+}
+
+#header {
+ background: $grad_start;
+ background: -webkit-gradient(linear, left top, left bottom, from($grad_start), to($grad_end)); /* for webkit browsers */
+ background: -moz-linear-gradient(top, $grad_start, $grad_end); /* for firefox 3.6+ */
+ height:110px;
+}
+
+#footer {
+ background: $grad_start;
+ background: -webkit-gradient(linear, left top, left bottom, from($grad_end), to($grad_start)); /* for webkit browsers */
+ background: -moz-linear-gradient(top, $grad_end, $grad_start); /* for firefox 3.6+ */
+ color:#555;
+ font-size:1em;
+ padding:2em;
+ text-align:center;
+}
+
+#header .container {
+ overflow: visible;
+ position:relative;
+}
+
+#header #logo {
+ left: 0;
+ position: absolute;
+ top: 25px;
+}
+
+#logo a {
+ border: none;
+}
+
+#logo a:hover{
+ text-decoration: none;
+ background: none;
+}
+
+#header img {
+ padding: 5px;
+}
+
+/* navbar */
+/*
+#navbar {
+ position: absolute;
+ right: 0;
+ top: 60px;
+}
+
+#navbar a {
+ border: none;
+ border-left:1px solid #DDDDDD;
+ font-size:1em;
+ padding:0.5em 1em;
+ text-decoration:none;
+ color: $sec_header;
+ background: none;
+}
+
+#navbar a.first {
+ border:0 none;
+}
+
+#navbar a:hover { text-decoration: none; }
+#navbar a:visited { text-decoration: none; }
+#navbar a:active { text-decoration: none; }
+#navbar a.active {
+ border-bottom: 3px solid #222;
+}
+
+*/
+
+.large {
+ font-size: 1.2em;
+ line-height:1.5;
+}
+
+.small {
+ font-size: 90%;
+ color: #555;
+ line-height: 1.2;
+}
+
+span.copyright {
+ text-shadow: 1px 1px 0 #FFFFFF;
+ color: #999;
+}
+
+span.high {
+ //color: green;
+}
+
+.titleheader {
+ font-size: 30px;
+ text-shadow: #fff 0px 1px 0;
+/* color: #C17878; */
+ color: $sec_header;
+ margin-bottom: 20px;
+}
+
+.secheader,h1 {
+ font-size: 25px;
+ text-shadow: #fff 0px 1px 0;
+/* color: #C17878; */
+ color: $sec_header;
+ margin-top: 20px;
+}
+
+.subsecheader,h2 {
+ font-size: 20px;
+ text-shadow: #fff 0px 1px 0;
+/* color: #C17878; */
+ color: $sec_header;
+ margin-top: 20px;
+}
+
+.subsecheader a {
+ color: $sec_header;
+}
+
+.subssubecheader,h3 {
+ font-size: 16px;
+ text-shadow: #fff 0px 1px 0;
+/* color: #C17878; */
+ color: $sec_header;
+ margin-top: 20px;
+}
+
+
+
+.horrule {
+ margin-top: 30px;
+ border-top: 1px solid #DDDDDD;
+ clear: both;
+}
+
+#os_logo {
+ height: 100px;
+ padding: 10px;
+}
+
+.sidetext {
+ border-left: 4px solid #ddd;
+ padding-left: 4px;
+}
+
+#download {
+ padding: 20px;
+ background: $dark_bg;
+ -moz-border-radius: 3px;
+ border-radius: 3px;
+ text-align: center;
+ border: 1px dotted #ddd;
+}
+
+#latest_head {
+ font-size: 14px;
+ text-shadow: #fff 0px 1px 0;
+ margin: 0;
+ color: $sec_header;
+}
+
+
+
+#latest_link {
+ background: #fefefe;
+ border: none;
+ padding: 3px;
+ margin-top: 10px;
+ -moz-border-radius: 5px;
+ border-radius: 5px;
+ font-weight: bold;
+ font-size: 80%;
+ -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, .4);
+ -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, .4);
+}
+
+#disclaimer {
+ color: #555;
+ margin-top: 20px;
+ font-size: 80%;
+}
+
+#latest_link a:hover {
+ background: none;
+}
+
+#maintainers {
+ margin-top: 20px;
+}
+
+.greybar {
+ border-left: 4px solid #ddd;
+ padding-left: 10px;
+}
+
+ul.greybar {
+ list-style-type: none;
+}
+
+#twitter_widget {
+ a { border: none;}
+ a:hover { border: none; background: inherit; text-decoration: none; }
+ a:visited { border: none; background: inherit; }
+ a:active { border: none; background: inherit; }
+}
+
+#twitter_follow {
+ a { border: none;}
+}
+
+.tabular table {
+ background: #fff;
+ border: 1px solid #ddd;
+}
+
+input[type="text"], input[type="password"], input.text, input.title, textarea, select {
+ margin: 1em 0;
+}
+
+input[type="text"], input[type="password"], input.text, input.title, textarea {
+ padding: 5px;
+ font-size: 14px;
+ background-color: #fff;
+ border: 1px solid #bbb;
+ color: #222;
+}
+
+#notice {
+ -moz-border-radius: 3px;
+ border-radius: 3px;
+ background: $dark_bg;
+ color: #444;
+ margin-bottom: 30px;
+ margin-right: 10px;
+ padding: 10px;
+ text-align: left;
+ // font-size: 14px;
+ -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, .4);
+ -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, .4);
+}
+
+#notice a {
+ color: #7E2217;
+}
\ No newline at end of file
diff --git a/website/content/team.haml b/website/content/team.haml
new file mode 100644
index 0000000..91a65c7
--- /dev/null
+++ b/website/content/team.haml
@@ -0,0 +1,42 @@
+---
+title: Team
+---
+#maintainers
+ %h2{:class=>"secheader"}= "maintainers"
+ %p= "Committers:"
+ .tabular
+ %table
+ %tr
+ %th= "name"
+ %th= "affiliation"
+ %tr
+ %td= "Kishore Gopalakrishna"
+ %td= "LinkedIn"
+ %tr
+ %td= "Flavio Junqueira"
+ %td= "Yahoo!"
+ %tr
+ %td= "Matthieu Morel"
+ %td= "Yahoo!"
+ %tr
+ %td= "Leo Neumeyer"
+ %td= "QuantBench"
+ %tr
+ %td= "Bruce Robbins"
+ %td= "Yahoo!"
+ %tr
+ %td= "Daniel Gomez Ferro"
+ %td= "Yahoo!"
+
+ %h2{:class=>"secheader"}= "former contributors"
+
+ .tabular
+ %table
+ %tr
+ %td= "Anand Kesari"
+ %td= "Yahoo!"
+ %tr
+ %td= "Anish Nair"
+ %td= "A9"
+
+ /%p= "We ask that you please do not send us emails privately asking for support. We are non-paid volunteers who help out with the project and we do not necessarily have the time or energy to help people on an individual basis. Instead, we have setup a mailing list for the project which can contain a wider range of individuals who will help answer detailed requests for help. The benefit of using mailing lists over private communication is that it is a shared resource where others can also learn from common mistakes and as a community we all grow together."
diff --git a/website/layouts/default.haml b/website/layouts/default.haml
new file mode 100644
index 0000000..7cee573
--- /dev/null
+++ b/website/layouts/default.haml
@@ -0,0 +1,26 @@
+!!!
+%html
+ %head
+ %title= "S4: #{@item[:title]}"
+ %meta{"http-equiv" => "Content-Type", :content=>"text/html; charset=utf-8"}
+ %meta{:name=>"description", :content=>"A general-purpose distributed stream computing platform"}
+ %link{:rel=>"stylesheet", :type=>"text/css", :href=>"/style/screen.css", :media=>"screen" }
+ %link{:rel=>"stylesheet", :type=>"text/css", :href=>"/style/print.css", :media=>"print" }
+ /[if lt IE 9]
+ %link{:rel=>"stylesheet", :type=>"text/css", :href=>"/style/ie.css", :media=>"screen" }
+ %link{:rel=>"stylesheet", :type=>"text/css", :href=>"/style/style.css"}
+ %link{:rel=>"stylesheet", :type=>"text/css", :href=>"/style/nav.css"}
+ = google_analytics(@config[:google_analytics_account_id], @config[:google_analytics_domain])
+ %body
+ #header
+ .container
+ #logo
+ %a{:href=>'/'}
+ %img{:src=>'/images/s4_test.png'}
+ #navbar= render 'nav'
+ #wrapper
+ #container{:class=>"container"}= yield
+ #footer
+ .container
+ %span{:class=>'copyright'}= "Apache S4 - Copyright 2013 The Apache Software Foundation"
+
diff --git a/website/layouts/nav.haml b/website/layouts/nav.haml
new file mode 100644
index 0000000..a8229b3
--- /dev/null
+++ b/website/layouts/nav.haml
@@ -0,0 +1,15 @@
+%ul{:id=>"nav"}
+ %li
+ %a{:href=>"/"}= "home"
+ %li
+ %a{:href=>"https://cwiki.apache.org/confluence/display/S4/S4+Wiki"}= "doc [0.5]"
+ %li
+ %a{:href=>"http://git-wip-us.apache.org/repos/asf?p=incubator-s4.git", :onClick=> google_analytics_event("External", "Apache Git", "http://github.com/s4")}= "code"
+ %li
+ %a{:href=>"http://people.apache.org/~mmorel/apache-s4-0.5.0-incubating-doc/javadoc/"}="API"
+ %li
+ %a{:href=>"/contrib"}= "get involved"
+ %li
+ %a{:href=>"/team"}= "team"
+ %li
+ %a{:href=>"/download"}= "download"
diff --git a/website/lib/default.rb b/website/lib/default.rb
new file mode 100644
index 0000000..967a0c7
--- /dev/null
+++ b/website/lib/default.rb
@@ -0,0 +1,68 @@
+# All files in the 'lib' directory will be loaded
+# before nanoc starts compiling.
+include Nanoc3::Helpers::Rendering
+
+def email(id, domain, label)
+ "<script language='JavaScript'>
+ <!-- // go away spammer!
+ var name = '#{id}';
+ var domain = '#{domain}';
+ var label = '#{label}';
+ document.write('<a class=\"email\" href=\"mailto:' + name + '@' + domain + '\">');
+ document.write(label + '</a>');
+ // -->
+ </script>"
+end
+
+def google_analytics(account_id, domain)
+ "<script type='text/javascript'>
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', '#{account_id}']);
+ _gaq.push(['_setDomainName', '#{domain}']);
+ _gaq.push(['_trackPageview']);
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+ </script>
+ "
+end
+
+def google_analytics_event(category, action, label)
+ "_gaq.push(['_trackEvent', '#{category}', '#{action}', '#{label}']);"
+end
+
+def twitter_widget()
+ "<script src='http://widgets.twimg.com/j/2/widget.js'></script>
+ <script>
+ new TWTR.Widget({
+ version: 2,
+ type: 'profile',
+ rpp: 4,
+ interval: 6000,
+ width: 190,
+ height: 300,
+ theme: {
+ shell: {
+ background: '#e6e6e6',
+ color: '#707070'
+ },
+ tweets: {
+ background: '#ffffff',
+ color: '#333333',
+ links: '#7E2217'
+ }
+ },
+ features: {
+ scrollbar: false,
+ loop: false,
+ live: false,
+ hashtags: true,
+ timestamp: true,
+ avatars: false,
+ behavior: 'all'
+ }
+ }).render().setUser('s4project').start();
+ </script>"
+end
\ No newline at end of file
diff --git a/website/resources/YCLA.pdf b/website/resources/YCLA.pdf
new file mode 100644
index 0000000..66b445b
--- /dev/null
+++ b/website/resources/YCLA.pdf
Binary files differ
