| |
| <!DOCTYPE html> |
| <html lang="en" dir=ZgotmplZ> |
| |
| <head> |
| |
| |
| |
| <link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css"> |
| <script src="/bootstrap/js/bootstrap.bundle.min.js"></script> |
| <link rel="stylesheet" type="text/css" href="/font-awesome/css/font-awesome.min.css"> |
| <script src="/js/anchor.min.js"></script> |
| <script src="/js/flink.js"></script> |
| <link rel="canonical" href="https://flink.apache.org/2020/08/06/accelerating-your-workload-with-gpu-and-other-external-resources/"> |
| |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| <meta name="description" content="Apache Flink 1.11 introduces a new External Resource Framework, which allows you to request external resources from the underlying resource management systems (e.g., Kubernetes) and accelerate your workload with those resources. As Flink provides a first-party GPU plugin at the moment, we will take GPU as an example and show how it affects Flink applications in the AI field. Other external resources (e.g. RDMA and SSD) can also be supported in a pluggable manner."> |
| <meta name="theme-color" content="#FFFFFF"><meta property="og:title" content="Accelerating your workload with GPU and other external resources" /> |
| <meta property="og:description" content="Apache Flink 1.11 introduces a new External Resource Framework, which allows you to request external resources from the underlying resource management systems (e.g., Kubernetes) and accelerate your workload with those resources. As Flink provides a first-party GPU plugin at the moment, we will take GPU as an example and show how it affects Flink applications in the AI field. Other external resources (e.g. RDMA and SSD) can also be supported in a pluggable manner." /> |
| <meta property="og:type" content="article" /> |
| <meta property="og:url" content="https://flink.apache.org/2020/08/06/accelerating-your-workload-with-gpu-and-other-external-resources/" /><meta property="article:section" content="posts" /> |
| <meta property="article:published_time" content="2020-08-06T08:00:00+00:00" /> |
| <meta property="article:modified_time" content="2020-08-06T08:00:00+00:00" /> |
| <title>Accelerating your workload with GPU and other external resources | Apache Flink</title> |
| <link rel="manifest" href="/manifest.json"> |
| <link rel="icon" href="/favicon.png" type="image/x-icon"> |
| <link rel="stylesheet" href="/book.min.22eceb4d17baa9cdc0f57345edd6f215a40474022dfee39b63befb5fb3c596b5.css" integrity="sha256-IuzrTRe6qc3A9XNF7dbyFaQEdAIt/uObY777X7PFlrU="> |
| <script defer src="/en.search.min.2698f0d1b683dae4d6cb071668b310a55ebcf1c48d11410a015a51d90105b53e.js" integrity="sha256-Jpjw0baD2uTWywcWaLMQpV688cSNEUEKAVpR2QEFtT4="></script> |
| <!-- |
| Made with Book Theme |
| https://github.com/alex-shpak/hugo-book |
| --> |
| |
| <meta name="generator" content="Hugo 0.124.1"> |
| |
| |
| <script> |
| var _paq = window._paq = window._paq || []; |
| |
| |
| _paq.push(['disableCookies']); |
| |
| _paq.push(["setDomains", ["*.flink.apache.org","*.nightlies.apache.org/flink"]]); |
| _paq.push(['trackPageView']); |
| _paq.push(['enableLinkTracking']); |
| (function() { |
| var u="//analytics.apache.org/"; |
| _paq.push(['setTrackerUrl', u+'matomo.php']); |
| _paq.push(['setSiteId', '1']); |
| var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; |
| g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); |
| })(); |
| </script> |
| |
| </head> |
| |
| <body dir=ZgotmplZ> |
| |
| |
| |
| <header> |
| <nav class="navbar navbar-expand-xl"> |
| <div class="container-fluid"> |
| <a class="navbar-brand" href="/"> |
| <img src="/img/logo/png/100/flink_squirrel_100_color.png" alt="Apache Flink" height="47" width="47" class="d-inline-block align-text-middle"> |
| <span>Apache Flink</span> |
| </a> |
| <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation"> |
| <i class="fa fa-bars navbar-toggler-icon"></i> |
| </button> |
| <div class="collapse navbar-collapse" id="navbarSupportedContent"> |
| <ul class="navbar-nav"> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="nav-item dropdown"> |
| <a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">About</a> |
| <ul class="dropdown-menu"> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/flink-architecture/">Architecture</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/flink-applications/">Applications</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/flink-operations/">Operations</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/use-cases/">Use Cases</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/powered-by/">Powered By</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/roadmap/">Roadmap</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/community/">Community & Project Info</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/security/">Security</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/what-is-flink/special-thanks/">Special Thanks</a> |
| |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| |
| |
| <li class="nav-item dropdown"> |
| <a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Getting Started</a> |
| <ul class="dropdown-menu"> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/try-flink/local_installation/">With Flink<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/docs/try-flink-kubernetes-operator/quick-start/">With Flink Kubernetes Operator<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable/docs/get-started/introduction/">With Flink CDC<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/docs/try-flink-ml/quick-start/">With Flink ML<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/getting-started/project-setup.html">With Flink Stateful Functions<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/learn-flink/overview/">Training Course<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| |
| |
| <li class="nav-item dropdown"> |
| <a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Documentation</a> |
| <ul class="dropdown-menu"> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/">Flink 1.19 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-master/">Flink Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/">Kubernetes Operator 1.8 (latest)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main">Kubernetes Operator Main (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable">CDC 3.0 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-master">CDC Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/">ML 2.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-master">ML Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/">Stateful Functions 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-master">Stateful Functions Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i> |
| </a> |
| |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| |
| |
| <li class="nav-item dropdown"> |
| <a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">How to Contribute</a> |
| <ul class="dropdown-menu"> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/overview/">Overview</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/contribute-code/">Contribute Code</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/reviewing-prs/">Review Pull Requests</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/code-style-and-quality-preamble/">Code Style and Quality Guide</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/contribute-documentation/">Contribute Documentation</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/documentation-style-guide/">Documentation Style Guide</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/improve-website/">Contribute to the Website</a> |
| |
| |
| </li> |
| |
| <li> |
| |
| |
| <a class="dropdown-item" href="/how-to-contribute/getting-help/">Getting Help</a> |
| |
| |
| </li> |
| |
| </ul> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| <li class="nav-item"> |
| |
| |
| <a class="nav-link" href="/posts/">Flink Blog</a> |
| |
| |
| </li> |
| |
| |
| |
| |
| |
| <li class="nav-item"> |
| |
| |
| <a class="nav-link" href="/downloads/">Downloads</a> |
| |
| |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| </ul> |
| <div class="book-search"> |
| <div class="book-search-spinner hidden"> |
| <i class="fa fa-refresh fa-spin"></i> |
| </div> |
| <form class="search-bar d-flex" onsubmit="return false;"su> |
| <input type="text" id="book-search-input" placeholder="Search" aria-label="Search" maxlength="64" data-hotkeys="s/"> |
| <i class="fa fa-search search"></i> |
| <i class="fa fa-circle-o-notch fa-spin spinner"></i> |
| </form> |
| <div class="book-search-spinner hidden"></div> |
| <ul id="book-search-results"></ul> |
| </div> |
| </div> |
| </div> |
| </nav> |
| <div class="navbar-clearfix"></div> |
| </header> |
| |
| |
| <main class="flex"> |
| <section class="container book-page"> |
| |
| <article class="markdown"> |
| <h1> |
| <a href="/2020/08/06/accelerating-your-workload-with-gpu-and-other-external-resources/">Accelerating your workload with GPU and other external resources</a> |
| </h1> |
| |
| |
| |
| August 6, 2020 - |
| |
| |
| |
| Yangze Guo |
| |
| |
| |
| |
| <p><p>Apache Flink 1.11 introduces a new <a href="//nightlies.apache.org/flink/flink-docs-master/ops/external_resources.html">External Resource Framework</a>, |
| which allows you to request external resources from the underlying resource management systems (e.g., Kubernetes) and accelerate your workload with |
| those resources. As Flink provides a first-party GPU plugin at the moment, we will take GPU as an example and show how it affects Flink applications |
| in the AI field. Other external resources (e.g. RDMA and SSD) can also be supported <a href="//nightlies.apache.org/flink/flink-docs-master/ops/external_resources.html#implement-a-plugin-for-your-custom-resource-type">in a pluggable manner</a>.</p> |
| <h1 id="end-to-end-real-time-ai-with-gpu"> |
| End-to-end real-time AI with GPU |
| <a class="anchor" href="#end-to-end-real-time-ai-with-gpu">#</a> |
| </h1> |
| <p>Recently, AI and Machine Learning have gained additional popularity and have been widely used in various scenarios, such |
| as personalized recommendation and image recognition. <a href="https://flink.apache.org/">Flink</a>, with the ability to support GPU |
| allocation, can be used to build an end-to-end real-time AI workflow.</p> |
| <h2 id="why-flink"> |
| Why Flink |
| <a class="anchor" href="#why-flink">#</a> |
| </h2> |
| <p>Typical AI workloads fall into two categories: training and inference.</p> |
| <center> |
| <img src="/img/blog/2020-08-06-accelerate-with-external-resources/ai-workflow.png" width="800px" alt="Typical AI Workflow"/> |
| <br/> |
| <i><small>Typical AI Workflow</small></i> |
| </center> |
| <br/> |
| <p>The training workload is usually a batch task, in which we train a model from a bounded dataset. On the other hand, the inference |
| workload tends to be a streaming job. It consumes an unbounded data stream, which contains image data, for example, and uses a model |
| to produce the output of predictions. Both workloads need to do data preprocessing first. Flink, as a |
| <a href="https://flink.apache.org/news/2019/02/13/unified-batch-streaming-blink.html">unified batch and stream processing engine</a>, can be used to build an end-to-end AI workflow naturally.</p> |
| <p>In many cases, the training and inference workload can benefit a lot by leveraging GPUs. <a href="https://azure.microsoft.com/en-us/blog/gpus-vs-cpus-for-deployment-of-deep-learning-models/">Research</a> |
| shows that CPU cluster is outperformed by GPU cluster, which is of similar cost, by about 400 percent. As training datasets |
| are getting bigger and models more complex, supporting GPUs has become mandatory for running AI workloads.</p> |
| <p>With the <a href="//nightlies.apache.org/flink/flink-docs-master/ops/external_resources.html">External Resource Framework</a> |
| and its <a href="//nightlies.apache.org/flink/flink-docs-master/ops/external_resources.html#plugin-for-gpu-resources">GPU plugin</a>, Flink |
| can now request GPU resources from the external resource management system and expose GPU information to operators. With this |
| feature, users can now easily build end-to-end training and real-time inference pipelines with GPU support on Flink.</p> |
| <h2 id="example-mnist-inference-with-flink"> |
| Example: MNIST Inference with Flink |
| <a class="anchor" href="#example-mnist-inference-with-flink">#</a> |
| </h2> |
| <p>We take the MNIST inference task as an example to show how to use the <a href="//nightlies.apache.org/flink/flink-docs-master/ops/external_resources.html">External Resource Framework</a> |
| and how to leverage GPUs in Flink. MNIST is a database of handwritten digits, which is usually viewed as the HelloWorld of AI. |
| The goal is to recognize a 28px*28px picture of a number from 0 to 9.</p> |
| <p>First, you need to set configurations for the external resource framework to enable GPU support:</p> |
| <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-bash" data-lang="bash"><span class="line"><span class="cl">external-resources: gpu |
| </span></span><span class="line"><span class="cl"><span class="c1"># Define the driver factory class of gpu resource.</span> |
| </span></span><span class="line"><span class="cl">external-resource.gpu.driver-factory.class: org.apache.flink.externalresource.gpu.GPUDriverFactory |
| </span></span><span class="line"><span class="cl"><span class="c1"># Define the amount of gpu resource per TaskManager.</span> |
| </span></span><span class="line"><span class="cl">external-resource.gpu.amount: <span class="m">1</span> |
| </span></span><span class="line"><span class="cl"><span class="c1"># Enable the coordination mode if you run it in standalone mode</span> |
| </span></span><span class="line"><span class="cl">external-resource.gpu.param.discovery-script.args: --enable-coordination |
| </span></span><span class="line"><span class="cl"> |
| </span></span><span class="line"><span class="cl"> |
| </span></span><span class="line"><span class="cl"><span class="c1"># If you run it on Yarn</span> |
| </span></span><span class="line"><span class="cl">external-resource.gpu.yarn.config-key: yarn.io/gpu |
| </span></span><span class="line"><span class="cl"><span class="c1"># If you run it on Kubernetes</span> |
| </span></span><span class="line"><span class="cl">external-resource.gpu.kubernetes.config-key: nvidia.com/gpu |
| </span></span></code></pre></div><p>For more details of the configuration, please refer to the <a href="//nightlies.apache.org/flink/flink-docs-release-1.11/ops/external_resources.html#configurations-1">official documentation</a>.</p> |
| <p>In the MNIST inference task, we first need to read the images and do data preprocessing. You can download <a href="http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz">training</a> |
| or <a href="http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz">testing</a> data from <a href="http://yann.lecun.com/exdb/mnist/">this site</a>. |
| We provide a simple <a href="https://github.com/KarmaGYZ/flink-mnist/blob/master/src/main/java/org/apache/flink/MNISTReader.java">MNISTReader</a>. |
| It will read the image data located in the provided file path and transform each image into a list of floating point numbers.</p> |
| <p>Then, we need a classifier to recognize those images. A one-layer pre-trained neural network, whose prediction accuracy is 92.14%, |
| is used in our classify operator. To leverage GPUs in order to accelerate the matrix-matrix multiplication, we use <a href="https://github.com/jcuda/jcuda">JCuda</a> |
| to call the native Cuda API. The prediction logic of the <a href="https://github.com/KarmaGYZ/flink-mnist/blob/master/src/main/java/org/apache/flink/MNISTClassifier.java">MNISTClassifier</a> is shown below.</p> |
| <div class="highlight"><pre tabindex="0" class="chroma"><code class="language-java" data-lang="java"><span class="line"><span class="cl"><span class="kd">class</span> <span class="nc">MNISTClassifier</span><span class="w"> </span><span class="kd">extends</span><span class="w"> </span><span class="n">RichMapFunction</span><span class="o"><</span><span class="n">List</span><span class="o"><</span><span class="n">Float</span><span class="o">></span><span class="p">,</span><span class="w"> </span><span class="n">Integer</span><span class="o">></span><span class="w"> </span><span class="p">{</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="nd">@Override</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="kd">public</span><span class="w"> </span><span class="kt">void</span><span class="w"> </span><span class="nf">open</span><span class="p">(</span><span class="n">Configuration</span><span class="w"> </span><span class="n">parameters</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="c1">// Get the GPU information and select the first GPU.</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="kd">final</span><span class="w"> </span><span class="n">Set</span><span class="o"><</span><span class="n">ExternalResourceInfo</span><span class="o">></span><span class="w"> </span><span class="n">externalResourceInfos</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">getRuntimeContext</span><span class="p">().</span><span class="na">getExternalResourceInfos</span><span class="p">(</span><span class="n">resourceName</span><span class="p">);</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="kd">final</span><span class="w"> </span><span class="n">Optional</span><span class="o"><</span><span class="n">String</span><span class="o">></span><span class="w"> </span><span class="n">firstIndexOptional</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">externalResourceInfos</span><span class="p">.</span><span class="na">iterator</span><span class="p">().</span><span class="na">next</span><span class="p">().</span><span class="na">getProperty</span><span class="p">(</span><span class="s">"index"</span><span class="p">);</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="c1">// Initialize JCublas with the selected GPU</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="n">JCuda</span><span class="p">.</span><span class="na">cudaSetDevice</span><span class="p">(</span><span class="n">Integer</span><span class="p">.</span><span class="na">parseInt</span><span class="p">(</span><span class="n">firstIndexOptional</span><span class="p">.</span><span class="na">get</span><span class="p">()));</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="n">JCublas</span><span class="p">.</span><span class="na">cublasInit</span><span class="p">();</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="p">}</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="nd">@Override</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="kd">public</span><span class="w"> </span><span class="n">Integer</span><span class="w"> </span><span class="nf">map</span><span class="p">(</span><span class="n">List</span><span class="o"><</span><span class="n">Float</span><span class="o">></span><span class="w"> </span><span class="n">value</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="c1">// Performs multiplication using JCublas. The matrixPointer points to our pre-trained model.</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="n">JCublas</span><span class="p">.</span><span class="na">cublasSgemv</span><span class="p">(</span><span class="sc">'n'</span><span class="p">,</span><span class="w"> </span><span class="n">DIMENSIONS</span><span class="p">.</span><span class="na">f1</span><span class="p">,</span><span class="w"> </span><span class="n">DIMENSIONS</span><span class="p">.</span><span class="na">f0</span><span class="p">,</span><span class="w"> </span><span class="n">1</span><span class="p">.</span><span class="na">0f</span><span class="p">,</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="n">matrixPointer</span><span class="p">,</span><span class="w"> </span><span class="n">DIMENSIONS</span><span class="p">.</span><span class="na">f1</span><span class="p">,</span><span class="w"> </span><span class="n">inputPointer</span><span class="p">,</span><span class="w"> </span><span class="n">1</span><span class="p">,</span><span class="w"> </span><span class="n">0</span><span class="p">.</span><span class="na">0f</span><span class="p">,</span><span class="w"> </span><span class="n">outputPointer</span><span class="p">,</span><span class="w"> </span><span class="n">1</span><span class="p">);</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="c1">// Read the result back from GPU.</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="n">JCublas</span><span class="p">.</span><span class="na">cublasGetVector</span><span class="p">(</span><span class="n">DIMENSIONS</span><span class="p">.</span><span class="na">f1</span><span class="p">,</span><span class="w"> </span><span class="n">Sizeof</span><span class="p">.</span><span class="na">FLOAT</span><span class="p">,</span><span class="w"> </span><span class="n">outputPointer</span><span class="p">,</span><span class="w"> </span><span class="n">1</span><span class="p">,</span><span class="w"> </span><span class="n">Pointer</span><span class="p">.</span><span class="na">to</span><span class="p">(</span><span class="n">output</span><span class="p">),</span><span class="w"> </span><span class="n">1</span><span class="p">);</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="kt">int</span><span class="w"> </span><span class="n">result</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">0</span><span class="p">;</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="k">for</span><span class="w"> </span><span class="p">(</span><span class="kt">int</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">0</span><span class="p">;</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="o"><</span><span class="w"> </span><span class="n">DIMENSIONS</span><span class="p">.</span><span class="na">f1</span><span class="p">;</span><span class="w"> </span><span class="o">++</span><span class="n">i</span><span class="p">)</span><span class="w"> </span><span class="p">{</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="n">result</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="n">output</span><span class="o">[</span><span class="n">i</span><span class="o">]</span><span class="w"> </span><span class="o">></span><span class="w"> </span><span class="n">output</span><span class="o">[</span><span class="n">result</span><span class="o">]</span><span class="w"> </span><span class="o">?</span><span class="w"> </span><span class="n">i</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="n">result</span><span class="p">;</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="p">}</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="k">return</span><span class="w"> </span><span class="n">result</span><span class="p">;</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="p">}</span><span class="w"> |
| </span></span></span><span class="line"><span class="cl"><span class="w"></span><span class="p">}</span><span class="w"> |
| </span></span></span></code></pre></div><p>The complete MNIST inference project can be found <a href="https://github.com/KarmaGYZ/flink-mnist">here</a>. In this project, we simply |
| print the inference result to <strong>STDOUT</strong>. In the actual production environment, you could also write the result to Elasticsearch or Kafka, for example.</p> |
| <p>The MNIST inference task is just a simple case that shows you how the external resource framework works and what Flink can |
| do with GPU support. With Flink’s open source extension <a href="https://github.com/alibaba/Alink">Alink</a>, which contains a lot of |
| pre-built algorithms based on Flink, and <a href="https://github.com/alibaba/flink-ai-extended">Tensorflow on Flink</a>, some complex |
| AI workloads, e.g. online learning, real-time inference service, could be easily implemented as well.</p> |
| <h1 id="other-external-resources"> |
| Other external resources |
| <a class="anchor" href="#other-external-resources">#</a> |
| </h1> |
| <p>In addition to GPU support, there are many other external resources that can be used to accelerate jobs in some specific scenarios. |
| E.g. FPGA, for AI workloads, is supported by both Yarn and Kubernetes. Some low-latency network devices, like RDMA and Solarflare, also |
| provide their device plugin for Kubernetes. Currently, Yarn supports GPUs and FPGAs, while the list of Kubernetes’ device plugins can be found <a href="https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/#examples">here</a>.</p> |
| <p>With the external resource framework, you only need to implement a plugin that enables the operator to get the information |
| for these external resources; see <a href="//nightlies.apache.org/flink/flink-docs-release-1.11/ops/external_resources.html#implement-a-plugin-for-your-custom-resource-type">Custom Plugin</a> |
| for more details. If you just want to ensure that an external resource exists in the TaskManager, then you only need to find the |
| configuration key of that resource in the underlying resource management system and configure the external resource framework accordingly.</p> |
| <h1 id="conclusion"> |
| Conclusion |
| <a class="anchor" href="#conclusion">#</a> |
| </h1> |
| <p>In the latest Flink release (Flink 1.11), an external resource framework has been introduced to support requesting various types of |
| resources from the underlying resource management systems, and supply all the necessary information for using these resources to the |
| operators. The first-party GPU plugin expands the application prospects of Flink in the AI domain. Different resource types can be supported |
| in a pluggable way. You can also implement your own plugins for custom resource types.</p> |
| <p>Future developments in this area include implementing operator level resource isolation and fine-grained external resource scheduling. |
| The community may kick this work off once <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-56%3A+Dynamic+Slot+Allocation">FLIP-56</a> |
| is finished. If you have any suggestions or questions for the community, we encourage you to sign up to the Apache Flink |
| <a href="https://flink.apache.org/community.html#mailing-lists">mailing lists</a> and join the discussion there.</p> |
| </p> |
| </article> |
| |
| |
| |
| |
| |
| |
| |
| <div class="edit-this-page"> |
| <p> |
| <a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications">Want to contribute translation?</a> |
| </p> |
| <p> |
| <a href="//github.com/apache/flink-web/edit/asf-site/docs/content/posts/2020-08-06-external-resource.md"> |
| Edit This Page<i class="fa fa-edit fa-fw"></i> |
| </a> |
| </p> |
| </div> |
| |
| </section> |
| |
| <aside class="book-toc"> |
| |
| |
| |
| <nav id="TableOfContents"><h3>On This Page <a href="javascript:void(0)" class="toc" onclick="collapseToc()"><i class="fa fa-times" aria-hidden="true"></i></a></h3> |
| <ul> |
| <li><a href="#end-to-end-real-time-ai-with-gpu">End-to-end real-time AI with GPU</a> |
| <ul> |
| <li><a href="#why-flink">Why Flink</a></li> |
| <li><a href="#example-mnist-inference-with-flink">Example: MNIST Inference with Flink</a></li> |
| </ul> |
| </li> |
| <li><a href="#other-external-resources">Other external resources</a></li> |
| <li><a href="#conclusion">Conclusion</a></li> |
| </ul> |
| </nav> |
| |
| |
| </aside> |
| <aside class="expand-toc hidden"> |
| <a class="toc" onclick="expandToc()" href="javascript:void(0)"> |
| <i class="fa fa-bars" aria-hidden="true"></i> |
| </a> |
| </aside> |
| |
| </main> |
| |
| <footer> |
| |
| |
| |
| <div class="separator"></div> |
| <div class="panels"> |
| <div class="wrapper"> |
| <div class="panel"> |
| <ul> |
| <li> |
| <a href="https://flink-packages.org/">flink-packages.org</a> |
| </li> |
| <li> |
| <a href="https://www.apache.org/">Apache Software Foundation</a> |
| </li> |
| <li> |
| <a href="https://www.apache.org/licenses/">License</a> |
| </li> |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| <li> |
| <a href="/zh/"> |
| <i class="fa fa-globe" aria-hidden="true"></i> 中文版 |
| </a> |
| </li> |
| |
| |
| |
| </ul> |
| </div> |
| <div class="panel"> |
| <ul> |
| <li> |
| <a href="/what-is-flink/security">Security</a--> |
| </li> |
| <li> |
| <a href="https://www.apache.org/foundation/sponsorship.html">Donate</a> |
| </li> |
| <li> |
| <a href="https://www.apache.org/foundation/thanks.html">Thanks</a> |
| </li> |
| </ul> |
| </div> |
| <div class="panel icons"> |
| <div> |
| <a href="/posts"> |
| <div class="icon flink-blog-icon"></div> |
| <span>Flink blog</span> |
| </a> |
| </div> |
| <div> |
| <a href="https://github.com/apache/flink"> |
| <div class="icon flink-github-icon"></div> |
| <span>Github</span> |
| </a> |
| </div> |
| <div> |
| <a href="https://twitter.com/apacheflink"> |
| <div class="icon flink-twitter-icon"></div> |
| <span>Twitter</span> |
| </a> |
| </div> |
| </div> |
| </div> |
| </div> |
| |
| <hr/> |
| |
| <div class="container disclaimer"> |
| <p>The contents of this website are © 2024 Apache Software Foundation under the terms of the Apache License v2. Apache Flink, Flink, and the Flink logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p> |
| </div> |
| |
| |
| |
| </footer> |
| |
| </body> |
| </html> |
| |
| |
| |
| |
| |
| |
