submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmarineServer.java - submarine - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  * KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations
  * under the License.
  */
 package org.apache.submarine.server;

 import org.apache.log4j.PropertyConfigurator;
 import org.apache.submarine.server.rest.provider.YamlEntityProvider;
 import org.apache.submarine.server.security.SecurityFactory;
 import org.apache.submarine.server.security.SecurityProvider;
 import org.apache.submarine.server.workbench.websocket.NotebookServer;
 import org.apache.submarine.server.websocket.WebSocketServer;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.SecureRequestCustomizer;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.SslConnectionFactory;
 import org.eclipse.jetty.server.handler.HandlerList;
 import org.eclipse.jetty.server.session.SessionHandler;
 import org.eclipse.jetty.servlet.DefaultServlet;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.eclipse.jetty.util.thread.ThreadPool;
 import org.eclipse.jetty.webapp.WebAppContext;
 import org.eclipse.jetty.websocket.servlet.WebSocketServlet;
 import org.glassfish.hk2.api.ServiceLocator;
 import org.glassfish.hk2.api.ServiceLocatorFactory;
 import org.glassfish.hk2.utilities.ServiceLocatorUtilities;
 import org.glassfish.hk2.utilities.binding.AbstractBinder;
 import org.glassfish.jersey.server.ResourceConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

 import org.apache.submarine.commons.utils.SubmarineConfiguration;
 import org.apache.submarine.commons.utils.SubmarineConfVars;

 import javax.inject.Inject;
 import javax.inject.Singleton;
 import javax.servlet.DispatcherType;
 import javax.servlet.Filter;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.EnumSet;
 import java.util.Optional;

 public class SubmarineServer extends ResourceConfig {
   private static final Logger LOG = LoggerFactory.getLogger(SubmarineServer.class);

   private static final long SERVERTIMESTAMP = System.currentTimeMillis();

   public static Server jettyWebServer;
   public static ServiceLocator sharedServiceLocator;
   private static WebAppContext webApp;
   private static final SubmarineConfiguration conf = SubmarineConfiguration.getInstance();

   public static long getServerTimeStamp() {
     return SERVERTIMESTAMP;
   }

   public static void main(String[] args) throws InterruptedException {
     PropertyConfigurator.configure(ClassLoader.getSystemResource("log4j.properties"));

     LOG.info("Submarine server Host: " + conf.getServerAddress());
     if (!conf.useSsl()) {
       LOG.info("Submarine server Port: " + conf.getServerPort());
     } else {
       LOG.info("Submarine server SSL Port: " + conf.getServerSslPort());
     }

     jettyWebServer = setupJettyServer(conf);

     // Web UI
     HandlerList handlers = new HandlerList();
     webApp = setupWebAppContext(handlers, conf);
     jettyWebServer.setHandler(handlers);

     // Add
     sharedServiceLocator = ServiceLocatorFactory.getInstance().create("shared-locator");
     ServiceLocatorUtilities.enableImmediateScope(sharedServiceLocator);
     ServiceLocatorUtilities.bind(
         sharedServiceLocator,
         new AbstractBinder() {
           @Override
           protected void configure() {
             bindAsContract(NotebookServer.class)
                 .to(WebSocketServlet.class)
                 .in(Singleton.class);
             bindAsContract(WebSocketServer.class)
                 .to(WebSocketServlet.class)
                 .in(Singleton.class);
           }
         });

     setupRestApiContextHandler(webApp, conf);

     // Notebook server
     setupNotebookServer(webApp, conf, sharedServiceLocator);

     // Cluster Server
     // Cluster Server is useless for submarine now. Shield it to improve performance.
     // setupClusterServer();

     setupWebSocketServer(webApp, conf, sharedServiceLocator);
     startServer();

   }

   @Inject
   public SubmarineServer() {
     packages("org.apache.submarine.server.workbench.rest",
         "org.apache.submarine.server.rest"
     );
     register(YamlEntityProvider.class);
   }

   private static void startServer() throws InterruptedException {
     LOG.info("Starting submarine server");
     try {
       // Instantiates SubmarineServer
       jettyWebServer.start();
     } catch (Exception e) {
       LOG.error("Error while running jettyServer", e);
       System.exit(-1);
     }
     LOG.info("Done, submarine server started");

     Runtime.getRuntime()
         .addShutdownHook(
             new Thread(
                 () -> {
                   LOG.info("Shutting down Submarine Server ... ");
                   try {
                     jettyWebServer.stop();
                     Thread.sleep(3000);
                   } catch (InterruptedException e) {
                     LOG.error("Interrupted exception:", e);
                   } catch (Exception e) {
                     LOG.error("Error while stopping servlet container", e);
                   }
                   LOG.info("Bye");
                 }));

     jettyWebServer.join();
   }

   private static void setupRestApiContextHandler(WebAppContext webapp, SubmarineConfiguration conf) {
     final ServletHolder servletHolder =
         new ServletHolder(new org.glassfish.jersey.servlet.ServletContainer());

     servletHolder.setInitParameter("javax.ws.rs.Application", SubmarineServer.class.getName());
     servletHolder.setName("rest");
     servletHolder.setForcedPath("rest");
     webapp.setSessionHandler(new SessionHandler());
     webapp.addServlet(servletHolder, "/api/*");
   }

   private static WebAppContext setupWebAppContext(HandlerList handlers,
                                                   SubmarineConfiguration conf) {
     WebAppContext webApp = new WebAppContext();
     webApp.setContextPath("/");
     File warPath = new File(conf.getString(SubmarineConfVars.ConfVars.WORKBENCH_WEB_WAR));
     LOG.info("workbench web war file path is {}.",
         conf.getString(SubmarineConfVars.ConfVars.WORKBENCH_WEB_WAR));
     if (warPath.isDirectory()) {
       // Development mode, read from FS
       webApp.setResourceBase(warPath.getPath());
       webApp.setParentLoaderPriority(true);
     } else {
       // use packaged WAR
       webApp.setWar(warPath.getAbsolutePath());
       File warTempDirectory = new File("webapps");
       warTempDirectory.mkdir();
       webApp.setTempDirectory(warTempDirectory);
     }

     webApp.addServlet(new ServletHolder(new DefaultServlet()), "/");
     // When requesting the workbench page, the content of index.html needs to be returned,
     // otherwise a 404 error will be displayed
     // NOTE: If you modify the workbench directory in the front-end URL,
     // you need to modify the `/workbench/*` here.
     webApp.addServlet(new ServletHolder(RefreshServlet.class), "/user/*");
     webApp.addServlet(new ServletHolder(RefreshServlet.class), "/workbench/*");

     // add security filter
     Optional<SecurityProvider> securityProvider = SecurityFactory.getSecurityProvider();
     if (securityProvider.isPresent()) {
       Class<Filter> filterClass = securityProvider.get().getFilterClass();
       LOG.info("Add {} to support auth", filterClass);
       webApp.addFilter(filterClass, "/*", EnumSet.of(DispatcherType.REQUEST));
     }

     handlers.setHandlers(new Handler[]{webApp});

     return webApp;
   }

   private static Server setupJettyServer(SubmarineConfiguration conf) {
     ThreadPool threadPool =
         new QueuedThreadPool(conf.getInt(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_MAX),
             conf.getInt(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_MIN),
             conf.getInt(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_TIMEOUT));
     final Server server = new Server(threadPool);
     ServerConnector connector;

     if (conf.useSsl()) {
       LOG.debug("Enabling SSL for submarine Server on port " + conf.getServerSslPort());
       HttpConfiguration httpConfig = new HttpConfiguration();
       httpConfig.setSecureScheme("https");
       httpConfig.setSecurePort(conf.getServerSslPort());
       httpConfig.setOutputBufferSize(32768);
       httpConfig.setResponseHeaderSize(8192);
       httpConfig.setSendServerVersion(true);

       HttpConfiguration httpsConfig = new HttpConfiguration(httpConfig);
       SecureRequestCustomizer src = new SecureRequestCustomizer();
       httpsConfig.addCustomizer(src);

       connector = new ServerConnector(
           server,
           new SslConnectionFactory(getSslContextFactory(conf), HttpVersion.HTTP_1_1.asString()),
           new HttpConnectionFactory(httpsConfig));
     } else {
       connector = new ServerConnector(server);
     }

     configureRequestHeaderSize(conf, connector);
     // Set some timeout options to make debugging easier.
     int timeout = 1000 * 30;
     connector.setIdleTimeout(timeout);
     connector.setHost(conf.getServerAddress());
     if (conf.useSsl()) {
       connector.setPort(conf.getServerSslPort());
     } else {
       connector.setPort(conf.getServerPort());
     }

     server.addConnector(connector);
     return server;
   }

   private static void setupNotebookServer(WebAppContext webapp,
                                           SubmarineConfiguration conf, ServiceLocator serviceLocator) {
     String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize();
     final ServletHolder servletHolder =
         new ServletHolder(serviceLocator.getService(NotebookServer.class));
     servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize);

     final ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
     webapp.addServlet(servletHolder, "/wss/*");
   }

   private static void setupWebSocketServer(WebAppContext webapp,
                                            SubmarineConfiguration conf, ServiceLocator serviceLocator) {
     String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize();
     final ServletHolder notebookServletHolder =
         new ServletHolder(serviceLocator.getService(WebSocketServer.class));
     notebookServletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize);

     final ServletHolder experimentServletHolder =
         new ServletHolder(serviceLocator.getService(WebSocketServer.class));
     experimentServletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize);

     final ServletHolder environmentServletHolder =
         new ServletHolder(serviceLocator.getService(WebSocketServer.class));
     environmentServletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize);


     final ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
     webapp.addServlet(notebookServletHolder, "/ws/notebook/*");
     webapp.addServlet(experimentServletHolder, "/ws/experiment/*");
     webapp.addServlet(environmentServletHolder, "/ws/environment/*");
   }

   private static SslContextFactory getSslContextFactory(SubmarineConfiguration conf) {
     SslContextFactory sslContextFactory = new SslContextFactory();

     // Set keystore
     sslContextFactory.setKeyStorePath(conf.getKeyStorePath());
     sslContextFactory.setKeyStoreType(conf.getKeyStoreType());
     sslContextFactory.setKeyStorePassword(conf.getKeyStorePassword());
     sslContextFactory.setKeyManagerPassword(conf.getKeyManagerPassword());

     if (conf.useClientAuth()) {
       sslContextFactory.setNeedClientAuth(conf.useClientAuth());

       // Set truststore
       sslContextFactory.setTrustStorePath(conf.getTrustStorePath());
       sslContextFactory.setTrustStoreType(conf.getTrustStoreType());
       sslContextFactory.setTrustStorePassword(conf.getTrustStorePassword());
     }

     return sslContextFactory;
   }

   private static void configureRequestHeaderSize(
       SubmarineConfiguration conf, ServerConnector connector) {
     HttpConnectionFactory cf =
         (HttpConnectionFactory) connector.getConnectionFactory(HttpVersion.HTTP_1_1.toString());
     int requestHeaderSize = conf.getJettyRequestHeaderSize();
     cf.getHttpConfiguration().setRequestHeaderSize(requestHeaderSize);
   }

   // SUBMARINE-422. Fix refreshing page returns 404 error
   // Because the workbench is developed using angular,
   // the adjustment of angular WEB pages is completely controlled by the front end,
   // so when you manually refresh a specific page in the browser,
   // the browser will send the request for this page to the back-end service,
   // but the back-end service only In response to API requests, it will cause the front end to display 404.
   // The solution is to find that not all API requests directly return the content of the index page,
   // so that the front end will automatically perform correct page routing processing.
   public static class RefreshServlet extends HttpServlet {
     private static final long serialVersionUID = 1L;

     @Override
     protected void doGet(HttpServletRequest request, HttpServletResponse response)
         throws ServletException, IOException {
       response.setContentType("text/html");
       response.encodeRedirectURL("/");
       response.setStatus(HttpServletResponse.SC_OK);

       File warPath = new File(conf.getString(SubmarineConfVars.ConfVars.WORKBENCH_WEB_WAR));
       File indexFile = null;
       if (warPath.isDirectory()) {
         // Development mode, read from FS
         indexFile = new File(warPath.getAbsolutePath() + "/index.html");
       } else {
         // Product mode, read from war file
         File warFile = webApp.getTempDirectory();
         if (!warFile.exists()) {
           throw new ServletException("Can't found war directory!");
         }
         indexFile = new File(warFile.getAbsolutePath() + "/webapp/index.html");
       }

       // If index.html does not exist, throw ServletException
       if (!(indexFile.isFile() && indexFile.exists())) {
         throw new ServletException("Can't found index html!");
       }

       StringBuilder sbIndexBuf = new StringBuilder();
       try (InputStreamReader reader =
                new InputStreamReader(new FileInputStream(indexFile), "GBK");
            BufferedReader bufferedReader = new BufferedReader(reader);) {
         String lineTxt = null;
         while ((lineTxt = bufferedReader.readLine()) != null) {
           sbIndexBuf.append(lineTxt);
         }
       } catch (IOException e) {
         LOG.error(e.getMessage(), e);
       }

       response.getWriter().print(sbIndexBuf.toString());
     }
   }
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing,
	* software distributed under the License is distributed on an
	* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	* KIND, either express or implied. See the License for the
	* specific language governing permissions and limitations
	* under the License.
	*/
	package org.apache.submarine.server;

	import org.apache.log4j.PropertyConfigurator;
	import org.apache.submarine.server.rest.provider.YamlEntityProvider;
	import org.apache.submarine.server.security.SecurityFactory;
	import org.apache.submarine.server.security.SecurityProvider;
	import org.apache.submarine.server.workbench.websocket.NotebookServer;
	import org.apache.submarine.server.websocket.WebSocketServer;
	import org.eclipse.jetty.http.HttpVersion;
	import org.eclipse.jetty.server.Handler;
	import org.eclipse.jetty.server.HttpConfiguration;
	import org.eclipse.jetty.server.HttpConnectionFactory;
	import org.eclipse.jetty.server.SecureRequestCustomizer;
	import org.eclipse.jetty.server.Server;
	import org.eclipse.jetty.server.ServerConnector;
	import org.eclipse.jetty.server.SslConnectionFactory;
	import org.eclipse.jetty.server.handler.HandlerList;
	import org.eclipse.jetty.server.session.SessionHandler;
	import org.eclipse.jetty.servlet.DefaultServlet;
	import org.eclipse.jetty.servlet.ServletContextHandler;
	import org.eclipse.jetty.servlet.ServletHolder;
	import org.eclipse.jetty.util.ssl.SslContextFactory;
	import org.eclipse.jetty.util.thread.QueuedThreadPool;
	import org.eclipse.jetty.util.thread.ThreadPool;
	import org.eclipse.jetty.webapp.WebAppContext;
	import org.eclipse.jetty.websocket.servlet.WebSocketServlet;
	import org.glassfish.hk2.api.ServiceLocator;
	import org.glassfish.hk2.api.ServiceLocatorFactory;
	import org.glassfish.hk2.utilities.ServiceLocatorUtilities;
	import org.glassfish.hk2.utilities.binding.AbstractBinder;
	import org.glassfish.jersey.server.ResourceConfig;
	import org.slf4j.Logger;
	import org.slf4j.LoggerFactory;

	import org.apache.submarine.commons.utils.SubmarineConfiguration;
	import org.apache.submarine.commons.utils.SubmarineConfVars;

	import javax.inject.Inject;
	import javax.inject.Singleton;
	import javax.servlet.DispatcherType;
	import javax.servlet.Filter;
	import javax.servlet.ServletException;
	import javax.servlet.http.HttpServlet;
	import javax.servlet.http.HttpServletRequest;
	import javax.servlet.http.HttpServletResponse;
	import java.io.BufferedReader;
	import java.io.File;
	import java.io.FileInputStream;
	import java.io.IOException;
	import java.io.InputStreamReader;
	import java.util.EnumSet;
	import java.util.Optional;

	public class SubmarineServer extends ResourceConfig {
	private static final Logger LOG = LoggerFactory.getLogger(SubmarineServer.class);

	private static final long SERVERTIMESTAMP = System.currentTimeMillis();

	public static Server jettyWebServer;
	public static ServiceLocator sharedServiceLocator;
	private static WebAppContext webApp;
	private static final SubmarineConfiguration conf = SubmarineConfiguration.getInstance();

	public static long getServerTimeStamp() {
	return SERVERTIMESTAMP;
	}

	public static void main(String[] args) throws InterruptedException {
	PropertyConfigurator.configure(ClassLoader.getSystemResource("log4j.properties"));

	LOG.info("Submarine server Host: " + conf.getServerAddress());
	if (!conf.useSsl()) {
	LOG.info("Submarine server Port: " + conf.getServerPort());
	} else {
	LOG.info("Submarine server SSL Port: " + conf.getServerSslPort());
	}

	jettyWebServer = setupJettyServer(conf);

	// Web UI
	HandlerList handlers = new HandlerList();
	webApp = setupWebAppContext(handlers, conf);
	jettyWebServer.setHandler(handlers);

	// Add
	sharedServiceLocator = ServiceLocatorFactory.getInstance().create("shared-locator");
	ServiceLocatorUtilities.enableImmediateScope(sharedServiceLocator);
	ServiceLocatorUtilities.bind(
	sharedServiceLocator,
	new AbstractBinder() {
	@Override
	protected void configure() {
	bindAsContract(NotebookServer.class)
	.to(WebSocketServlet.class)
	.in(Singleton.class);
	bindAsContract(WebSocketServer.class)
	.to(WebSocketServlet.class)
	.in(Singleton.class);
	}
	});

	setupRestApiContextHandler(webApp, conf);

	// Notebook server
	setupNotebookServer(webApp, conf, sharedServiceLocator);

	// Cluster Server
	// Cluster Server is useless for submarine now. Shield it to improve performance.
	// setupClusterServer();

	setupWebSocketServer(webApp, conf, sharedServiceLocator);
	startServer();

	}

	@Inject
	public SubmarineServer() {
	packages("org.apache.submarine.server.workbench.rest",
	"org.apache.submarine.server.rest"
	);
	register(YamlEntityProvider.class);
	}

	private static void startServer() throws InterruptedException {
	LOG.info("Starting submarine server");
	try {
	// Instantiates SubmarineServer
	jettyWebServer.start();
	} catch (Exception e) {
	LOG.error("Error while running jettyServer", e);
	System.exit(-1);
	}
	LOG.info("Done, submarine server started");

	Runtime.getRuntime()
	.addShutdownHook(
	new Thread(
	() -> {
	LOG.info("Shutting down Submarine Server ... ");
	try {
	jettyWebServer.stop();
	Thread.sleep(3000);
	} catch (InterruptedException e) {
	LOG.error("Interrupted exception:", e);
	} catch (Exception e) {
	LOG.error("Error while stopping servlet container", e);
	}
	LOG.info("Bye");
	}));

	jettyWebServer.join();
	}

	private static void setupRestApiContextHandler(WebAppContext webapp, SubmarineConfiguration conf) {
	final ServletHolder servletHolder =
	new ServletHolder(new org.glassfish.jersey.servlet.ServletContainer());

	servletHolder.setInitParameter("javax.ws.rs.Application", SubmarineServer.class.getName());
	servletHolder.setName("rest");
	servletHolder.setForcedPath("rest");
	webapp.setSessionHandler(new SessionHandler());
	webapp.addServlet(servletHolder, "/api/*");
	}

	private static WebAppContext setupWebAppContext(HandlerList handlers,
	SubmarineConfiguration conf) {
	WebAppContext webApp = new WebAppContext();
	webApp.setContextPath("/");
	File warPath = new File(conf.getString(SubmarineConfVars.ConfVars.WORKBENCH_WEB_WAR));
	LOG.info("workbench web war file path is {}.",
	conf.getString(SubmarineConfVars.ConfVars.WORKBENCH_WEB_WAR));
	if (warPath.isDirectory()) {
	// Development mode, read from FS
	webApp.setResourceBase(warPath.getPath());
	webApp.setParentLoaderPriority(true);
	} else {
	// use packaged WAR
	webApp.setWar(warPath.getAbsolutePath());
	File warTempDirectory = new File("webapps");
	warTempDirectory.mkdir();
	webApp.setTempDirectory(warTempDirectory);
	}

	webApp.addServlet(new ServletHolder(new DefaultServlet()), "/");
	// When requesting the workbench page, the content of index.html needs to be returned,
	// otherwise a 404 error will be displayed
	// NOTE: If you modify the workbench directory in the front-end URL,
	// you need to modify the `/workbench/*` here.
	webApp.addServlet(new ServletHolder(RefreshServlet.class), "/user/*");
	webApp.addServlet(new ServletHolder(RefreshServlet.class), "/workbench/*");

	// add security filter
	Optional<SecurityProvider> securityProvider = SecurityFactory.getSecurityProvider();
	if (securityProvider.isPresent()) {
	Class<Filter> filterClass = securityProvider.get().getFilterClass();
	LOG.info("Add {} to support auth", filterClass);
	webApp.addFilter(filterClass, "/*", EnumSet.of(DispatcherType.REQUEST));
	}

	handlers.setHandlers(new Handler[]{webApp});

	return webApp;
	}

	private static Server setupJettyServer(SubmarineConfiguration conf) {
	ThreadPool threadPool =
	new QueuedThreadPool(conf.getInt(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_MAX),
	conf.getInt(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_MIN),
	conf.getInt(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_TIMEOUT));
	final Server server = new Server(threadPool);
	ServerConnector connector;

	if (conf.useSsl()) {
	LOG.debug("Enabling SSL for submarine Server on port " + conf.getServerSslPort());
	HttpConfiguration httpConfig = new HttpConfiguration();
	httpConfig.setSecureScheme("https");
	httpConfig.setSecurePort(conf.getServerSslPort());
	httpConfig.setOutputBufferSize(32768);
	httpConfig.setResponseHeaderSize(8192);
	httpConfig.setSendServerVersion(true);

	HttpConfiguration httpsConfig = new HttpConfiguration(httpConfig);
	SecureRequestCustomizer src = new SecureRequestCustomizer();
	httpsConfig.addCustomizer(src);

	connector = new ServerConnector(
	server,
	new SslConnectionFactory(getSslContextFactory(conf), HttpVersion.HTTP_1_1.asString()),
	new HttpConnectionFactory(httpsConfig));
	} else {
	connector = new ServerConnector(server);
	}

	configureRequestHeaderSize(conf, connector);
	// Set some timeout options to make debugging easier.
	int timeout = 1000 * 30;
	connector.setIdleTimeout(timeout);
	connector.setHost(conf.getServerAddress());
	if (conf.useSsl()) {
	connector.setPort(conf.getServerSslPort());
	} else {
	connector.setPort(conf.getServerPort());
	}

	server.addConnector(connector);
	return server;
	}

	private static void setupNotebookServer(WebAppContext webapp,
	SubmarineConfiguration conf, ServiceLocator serviceLocator) {
	String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize();
	final ServletHolder servletHolder =
	new ServletHolder(serviceLocator.getService(NotebookServer.class));
	servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize);

	final ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
	webapp.addServlet(servletHolder, "/wss/*");
	}

	private static void setupWebSocketServer(WebAppContext webapp,
	SubmarineConfiguration conf, ServiceLocator serviceLocator) {
	String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize();
	final ServletHolder notebookServletHolder =
	new ServletHolder(serviceLocator.getService(WebSocketServer.class));
	notebookServletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize);

	final ServletHolder experimentServletHolder =
	new ServletHolder(serviceLocator.getService(WebSocketServer.class));
	experimentServletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize);

	final ServletHolder environmentServletHolder =
	new ServletHolder(serviceLocator.getService(WebSocketServer.class));
	environmentServletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize);



	final ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
	webapp.addServlet(notebookServletHolder, "/ws/notebook/*");
	webapp.addServlet(experimentServletHolder, "/ws/experiment/*");
	webapp.addServlet(environmentServletHolder, "/ws/environment/*");
	}

	private static SslContextFactory getSslContextFactory(SubmarineConfiguration conf) {
	SslContextFactory sslContextFactory = new SslContextFactory();

	// Set keystore
	sslContextFactory.setKeyStorePath(conf.getKeyStorePath());
	sslContextFactory.setKeyStoreType(conf.getKeyStoreType());
	sslContextFactory.setKeyStorePassword(conf.getKeyStorePassword());
	sslContextFactory.setKeyManagerPassword(conf.getKeyManagerPassword());

	if (conf.useClientAuth()) {
	sslContextFactory.setNeedClientAuth(conf.useClientAuth());

	// Set truststore
	sslContextFactory.setTrustStorePath(conf.getTrustStorePath());
	sslContextFactory.setTrustStoreType(conf.getTrustStoreType());
	sslContextFactory.setTrustStorePassword(conf.getTrustStorePassword());
	}

	return sslContextFactory;
	}

	private static void configureRequestHeaderSize(
	SubmarineConfiguration conf, ServerConnector connector) {
	HttpConnectionFactory cf =
	(HttpConnectionFactory) connector.getConnectionFactory(HttpVersion.HTTP_1_1.toString());
	int requestHeaderSize = conf.getJettyRequestHeaderSize();
	cf.getHttpConfiguration().setRequestHeaderSize(requestHeaderSize);
	}

	// SUBMARINE-422. Fix refreshing page returns 404 error
	// Because the workbench is developed using angular,
	// the adjustment of angular WEB pages is completely controlled by the front end,
	// so when you manually refresh a specific page in the browser,
	// the browser will send the request for this page to the back-end service,
	// but the back-end service only In response to API requests, it will cause the front end to display 404.
	// The solution is to find that not all API requests directly return the content of the index page,
	// so that the front end will automatically perform correct page routing processing.
	public static class RefreshServlet extends HttpServlet {
	private static final long serialVersionUID = 1L;

	@Override
	protected void doGet(HttpServletRequest request, HttpServletResponse response)
	throws ServletException, IOException {
	response.setContentType("text/html");
	response.encodeRedirectURL("/");
	response.setStatus(HttpServletResponse.SC_OK);

	File warPath = new File(conf.getString(SubmarineConfVars.ConfVars.WORKBENCH_WEB_WAR));
	File indexFile = null;
	if (warPath.isDirectory()) {
	// Development mode, read from FS
	indexFile = new File(warPath.getAbsolutePath() + "/index.html");
	} else {
	// Product mode, read from war file
	File warFile = webApp.getTempDirectory();
	if (!warFile.exists()) {
	throw new ServletException("Can't found war directory!");
	}
	indexFile = new File(warFile.getAbsolutePath() + "/webapp/index.html");
	}

	// If index.html does not exist, throw ServletException
	if (!(indexFile.isFile() && indexFile.exists())) {
	throw new ServletException("Can't found index html!");
	}

	StringBuilder sbIndexBuf = new StringBuilder();
	try (InputStreamReader reader =
	new InputStreamReader(new FileInputStream(indexFile), "GBK");
	BufferedReader bufferedReader = new BufferedReader(reader);) {
	String lineTxt = null;
	while ((lineTxt = bufferedReader.readLine()) != null) {
	sbIndexBuf.append(lineTxt);
	}
	} catch (IOException e) {
	LOG.error(e.getMessage(), e);
	}

	response.getWriter().print(sbIndexBuf.toString());
	}
	}
	}