/****************************************************************************
 * apps/netutils/webclient/webclient.c
 *
 * SPDX-License-Identifier: BSD-3-Clause
 * SPDX-FileCopyrightText: 2014, 2020 Gregory Nutt. All rights reserved.
 * SPDX-FileCopyrightText: 2011-2012 Gregory Nutt. All rights reserved.
 * SPDX-FileCopyrightText: 2007, 2009 Gregory Nutt. All rights reserved.
 * SPDX-FileCopyrightText: 2002 Adam Dunkels. All rights reserved.
 * SPDX-FileContributor: Gregory Nutt <gnutt@nuttx.org>
 * SPDX-FileContributor: Adam Dunkels <adam@dunkels.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above
 *    copyright notice, this list of conditions and the following
 *    disclaimer in the documentation and/or other materials provided
 *    with the distribution.
 * 3. The name of the author may not be used to endorse or promote
 *    products derived from this software without specific prior
 *    written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 ****************************************************************************/

/* This example shows a HTTP client that is able to download web pages
 * and files from web servers. It requires a number of callback
 * functions to be implemented by the module that utilizes the code:
 * webclient_datahandler().
 */

/****************************************************************************
 * Included Files
 ****************************************************************************/

#include <nuttx/config.h>
#include <nuttx/compiler.h>
#include <debug.h>

#include <assert.h>
#include <fcntl.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <stdint.h>
#include <stdio.h>
#include <stdbool.h>
#include <unistd.h>
#include <netdb.h>
#include <string.h>
#include <strings.h>
#include <stdlib.h>
#include <errno.h>
#include <inttypes.h>

#include <arpa/inet.h>
#include <netinet/in.h>
#if defined(CONFIG_WEBCLIENT_NET_LOCAL)
#include <sys/un.h>
#endif

#include <nuttx/version.h>

#include "netutils/netlib.h"
#include "netutils/webclient.h"

#if defined(CONFIG_NETUTILS_CODECS)
#  if defined(CONFIG_CODECS_URLCODE)
#    include "netutils/urldecode.h"
#  endif
#  if defined(CONFIG_CODECS_BASE64)
#    include "netutils/base64.h"
#  endif
#else
#  undef CONFIG_CODECS_URLCODE
#  undef CONFIG_CODECS_BASE64
#endif

#ifndef CONFIG_NSH_WGET_USERAGENT
#  if CONFIG_VERSION_MAJOR != 0 || CONFIG_VERSION_MINOR != 0
#    define CONFIG_NSH_WGET_USERAGENT \
     "NuttX/" CONFIG_VERSION_STRING " (; http://www.nuttx.org/)"
#  else
#    define CONFIG_NSH_WGET_USERAGENT \
    "NuttX/6.xx.x (; http://www.nuttx.org/)"
#  endif
#endif

/****************************************************************************
 * Pre-processor Definitions
 ****************************************************************************/

#ifndef CONFIG_WEBCLIENT_TIMEOUT
#  define CONFIG_WEBCLIENT_TIMEOUT 10
#endif

#ifndef CONFIG_WEBCLIENT_MAX_REDIRECT
/* The default value 50 is taken from curl's --max-redirs option. */
#  define CONFIG_WEBCLIENT_MAX_REDIRECT 50
#endif

#define HTTPSTATUS_NONE            0
#define HTTPSTATUS_OK              1
#define HTTPSTATUS_MOVED           2
#define HTTPSTATUS_ERROR           3

#define ISO_NL                     0x0a
#define ISO_CR                     0x0d
#define ISO_SPACE                  0x20

#define WGET_MODE_GET              0
#define WGET_MODE_POST             1

/* The following CONN_ flags are for conn::flags.
 */

#define CONN_WANT_READ  WEBCLIENT_POLL_INFO_WANT_READ
#define CONN_WANT_WRITE WEBCLIENT_POLL_INFO_WANT_WRITE

#ifdef CONFIG_DEBUG_ASSERTIONS
#define _CHECK_STATE(ctx, s) DEBUGASSERT((ctx)->state == (s))
#define _SET_STATE(ctx, s)   ctx->state = (s)
#else
#define _CHECK_STATE(ctx, s) do {} while (0)
#define _SET_STATE(ctx, s)   do {} while (0)
#endif

/****************************************************************************
 * Private Types
 ****************************************************************************/

enum webclient_state_e
  {
    WEBCLIENT_STATE_SOCKET,
    WEBCLIENT_STATE_CONNECT,
    WEBCLIENT_STATE_PREPARE_REQUEST,
    WEBCLIENT_STATE_SEND_REQUEST,
    WEBCLIENT_STATE_SEND_REQUEST_BODY,
    WEBCLIENT_STATE_STATUSLINE,
    WEBCLIENT_STATE_HEADERS,
    WEBCLIENT_STATE_DATA,
    WEBCLIENT_STATE_CHUNKED_HEADER,
    WEBCLIENT_STATE_CHUNKED_DATA,
    WEBCLIENT_STATE_CHUNKED_ENDDATA,
    WEBCLIENT_STATE_CHUNKED_TRAILER,
    WEBCLIENT_STATE_WAIT_CLOSE,
    WEBCLIENT_STATE_CLOSE,
    WEBCLIENT_STATE_DONE,
    WEBCLIENT_STATE_TUNNEL_ESTABLISHED,
  };

/* flags for wget_s::internal_flags */

#define WGET_FLAG_GOT_CONTENT_LENGTH 1U
#define WGET_FLAG_CHUNKED            2U
#define WGET_FLAG_GOT_LOCATION       4U

struct wget_target_s
{
  char scheme[sizeof("https") + 1];
  char hostname[CONFIG_WEBCLIENT_MAXHOSTNAME];
  char filename[CONFIG_WEBCLIENT_MAXFILENAME];
  uint16_t port;     /* The port number to use in the connection */
};

struct wget_s
{
  /* Internal status */

  enum webclient_state_e state;
  uint8_t httpstatus;

  /* These describe the just-received buffer of data */

  FAR char *buffer;  /* user-provided buffer */
  int buflen;        /* Length of the user provided buffer */
  int offset;        /* Offset to the beginning of interesting data */
  int datend;        /* Offset+1 to the last valid byte of data in the buffer */

  /* Buffer HTTP header data and parse line at a time */

  char line[CONFIG_WEBCLIENT_MAXHTTPLINE];
  int  ndx;
  bool skip_to_next_line;

  unsigned int internal_flags; /* OR'ed WGET_FLAG_xxx */
  uintmax_t expected_resp_body_len;
  uintmax_t received_body_len;

  uintmax_t chunk_len;
  uintmax_t chunk_received;

#ifdef CONFIG_WEBCLIENT_GETMIMETYPE
  char mimetype[CONFIG_WEBCLIENT_MAXMIMESIZE];
#endif

  struct wget_target_s target;
  struct wget_target_s proxy;

  bool need_conn_close;
  struct webclient_conn_s *conn;
  unsigned int nredirect;
  int redirected;

  /* progress and todo for the current state (WEBCLIENT_STATE_) */

  off_t state_offset;
  size_t state_len;
  FAR const void *data_buffer;
  size_t data_len;

  FAR struct webclient_context *tunnel;
};

/****************************************************************************
 * Private Data
 ****************************************************************************/

static const char g_http10[]               = "HTTP/1.0";
static const char g_http11[]               = "HTTP/1.1";
#ifdef CONFIG_WEBCLIENT_GETMIMETYPE
static const char g_httpcontenttype[]      = "content-type: ";
#endif
static const char g_httphost[]             = "host: ";
static const char g_httplocation[]         = "location: ";
static const char g_httptransferencoding[] = "transfer-encoding: ";

static const char g_httpuseragentfields[] =
  "User-Agent: "
  CONFIG_NSH_WGET_USERAGENT
  "\r\n\r\n";

static const char g_httpcrnl[]       = "\r\n";

static const char g_httpform[]       = "Content-Type: "
                                       "application/x-www-form-urlencoded";
static const char g_httpcontsize[]   = "Content-Length: ";
static const char g_httpconn_close[] = "Connection: close";
#if 0
static const char g_httpconn[]       = "Connection: Keep-Alive";
static const char g_httpcache[]      = "Cache-Control: no-cache";
#endif

/****************************************************************************
 * Private Functions
 ****************************************************************************/

/****************************************************************************
 * Name: free_ws
 ****************************************************************************/

static void free_ws(FAR struct wget_s *ws)
{
  if (ws->conn != NULL)
    {
      webclient_conn_free(ws->conn);
    }

  free(ws->tunnel);
  free(ws);
}

/****************************************************************************
 * Name: webclient_static_body_func
 ****************************************************************************/

static int webclient_static_body_func(FAR void *buffer,
                                      FAR size_t *sizep,
                                      FAR const void * FAR *datap,
                                      size_t reqsize,
                                      FAR void *ctx)
{
  UNUSED(buffer);
  *datap = ctx;
  *sizep = reqsize;
  return 0;
}

/****************************************************************************
 * Name: append
 ****************************************************************************/

static char *append(char *dest, char *ep, const char *src)
{
  int len;

  if (dest == NULL)
    {
      return NULL;
    }

  len = strlen(src);
  if (ep - dest < len + 1)
    {
      return NULL;
    }

  memcpy(dest, src, len);
  dest[len] = '\0';
  return dest + len;
}

/****************************************************************************
 * Name: wget_strcpy
 ****************************************************************************/

#ifdef WGET_USE_URLENCODE
static char *wget_strcpy(char *dest, const char *src)
{
  int len = strlen(src);

  memcpy(dest, src, len);
  dest[len] = '\0';
  return dest + len;
}
#endif

/****************************************************************************
 * Name: wget_urlencode_strcpy
 ****************************************************************************/

#ifdef WGET_USE_URLENCODE
static char *wget_urlencode_strcpy(char *dest, const char *src)
{
  int len = strlen(src);
  int d_len;

  d_len = urlencode_len(src, len);
  urlencode(src, len, dest, &d_len);
  return dest + d_len;
}
#endif

/****************************************************************************
 * Name: wget_parseint
 ****************************************************************************/

static int wget_parseint(const char *cp, uintmax_t *resultp, int base)
{
  char *ep;
  uintmax_t val;

  errno = 0;
  val = strtoumax(cp, &ep, base);
  if (cp == ep)
    {
      return -EINVAL; /* not a number */
    }

  if (*ep != '\0')
    {
      return -EINVAL; /* not a number */
    }

  if (errno != 0)
    {
      DEBUGASSERT(errno == ERANGE);
      DEBUGASSERT(val == UINTMAX_MAX);
      return -errno;
    }

  *resultp = val;
  return 0;
}

/****************************************************************************
 * Name: wget_parsestatus
 ****************************************************************************/

static inline int wget_parsestatus(struct webclient_context *ctx,
                                   struct wget_s *ws)
{
  int offset;
  int ndx;
  char *dest;

  offset = ws->offset;
  ndx    = ws->ndx;

  while (offset < ws->datend)
    {
      bool got_nl;

      ws->line[ndx] = ws->buffer[offset++];
      got_nl = ws->line[ndx] == ISO_NL;
      if (got_nl || ndx == CONFIG_WEBCLIENT_MAXHTTPLINE - 1)
        {
          if (!got_nl)
            {
              nerr("ERROR: HTTP status line didn't fit "
                   "CONFIG_WEBCLIENT_MAXHTTPLINE: %.*s\n",
                   ndx, ws->line);
              return -E2BIG;
            }

          /* HTTP status line is something like:
           *
           * HTTP/1.1 200 OK
           *
           * https://datatracker.ietf.org/doc/html/rfc7230#section-3.1.2
           *
           * > status-line = HTTP-version SP status-code \
           * >               SP reason-phrase CRLF
           */

          ws->line[ndx] = '\0';
          if ((strncmp(ws->line, g_http10, strlen(g_http10)) == 0) ||
              (strncmp(ws->line, g_http11, strlen(g_http11)) == 0))
            {
              unsigned long http_status;
              char *ep;

              DEBUGASSERT(strlen(g_http10) == 8);
              DEBUGASSERT(strlen(g_http11) == 8);

              /* SP before the status-code */

              if (ws->line[8] != ' ')
                {
                  return -EINVAL;
                }

              dest = &(ws->line[9]);  /* the status-code */
              ws->httpstatus = HTTPSTATUS_NONE;

              errno = 0;
              http_status = strtoul(dest, &ep, 10);
              if (ep == dest)
                {
                  return -EINVAL;
                }

              /* SP before reason-phrase */

              if (*ep != ' ')
                {
                  return -EINVAL;
                }

              if (errno != 0)
                {
                  return -errno;
                }

              ctx->http_status = http_status;
              ninfo("Got HTTP status %lu\n", http_status);
              if (ctx->http_reason != NULL)
                {
                  strlcpy(ctx->http_reason,
                          ep + 1,
                          ctx->http_reason_len);
                }

              /* Check for 2xx (Successful) */

              if ((http_status / 100) == 2)
                {
                  ws->httpstatus = HTTPSTATUS_OK;
                }

              /* Check for 3xx (Redirection)
               * Location: header line will contain the new location.
               *
               * Note: the following 3xx are not redirects.
               *   304 not modified
               *   305 use proxy
               */

              else if ((http_status / 100) == 3 &&
                       (http_status != 304) &&
                       (http_status != 305))
                {
                  ws->httpstatus = HTTPSTATUS_MOVED;
                }
            }
          else
            {
              return -ECONNABORTED;
            }

          /* We're done parsing the status line, so start parsing
           * the HTTP headers.
           */

          ws->state = WEBCLIENT_STATE_HEADERS;
          ws->internal_flags &= ~(WGET_FLAG_GOT_CONTENT_LENGTH |
                                  WGET_FLAG_CHUNKED |
                                  WGET_FLAG_GOT_LOCATION);
          ndx = 0;
          break;
        }
      else
        {
          ndx++;
        }
    }

  ws->offset = offset;
  ws->ndx    = ndx;
  return OK;
}

/****************************************************************************
 * Name: parseurl
 ****************************************************************************/

static int parseurl(FAR const char *url, FAR struct wget_target_s *targ,
                    bool require_port)
{
  struct url_s url_s;
  int ret;

  memset(&url_s, 0, sizeof(url_s));
  url_s.scheme = targ->scheme;
  url_s.schemelen = sizeof(targ->scheme);
  url_s.host = targ->hostname;
  url_s.hostlen = sizeof(targ->hostname);
  url_s.path = targ->filename;
  url_s.pathlen = sizeof(targ->filename);
  ret = netlib_parseurl(url, &url_s);
  if (ret < 0)
    {
      return ret;
    }

  if (url_s.port == 0)
    {
      if (require_port)
        {
          return -EINVAL;
        }

      if (!strcmp(targ->scheme, "https"))
        {
          targ->port = 443;
        }
      else
        {
          targ->port = 80;
        }
    }
  else
    {
      targ->port = url_s.port;
    }

  return 0;
}

/****************************************************************************
 * Name: wget_parseheaders
 ****************************************************************************/

static inline int wget_parseheaders(struct webclient_context *ctx,
                                    struct wget_s *ws)
{
  int offset;
  int ndx;
  int ret = OK;

  offset = ws->offset;
  ndx    = ws->ndx;

  while (offset < ws->datend)
    {
      bool got_nl;

      ws->line[ndx] = ws->buffer[offset++];
      got_nl = ws->line[ndx] == ISO_NL;
      if (got_nl || ndx == CONFIG_WEBCLIENT_MAXHTTPLINE - 1)
        {
          bool found;

          if (ws->skip_to_next_line)
            {
              if (got_nl)
                {
                  ws->skip_to_next_line = false;
                }

              ndx = 0;
              continue;
            }

          /* We have an entire HTTP header line in ws->line, or
           * our buffer is already full, so we start parsing it.
           */

          found = false;

          /* Should always be true */

          if (ndx > 0)
            {
              ninfo("Got HTTP header line%s: %.*s\n",
                    got_nl ? "" : " (truncated)",
                    ndx - 1, &ws->line[0]);

              if (ws->line[0] == ISO_CR)
                {
                  /* This was the last header line (i.e., and empty "\r\n"),
                   * so we are done with the headers and proceed with the
                   * actual data.
                   */

                  if ((ws->internal_flags & WGET_FLAG_CHUNKED) != 0)
                    {
                      ws->state = WEBCLIENT_STATE_CHUNKED_HEADER;
                      ndx = 0;
                    }
                  else
                    {
                      if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) != 0)
                        {
                          if (ctx->http_status / 100 == 2)
                            {
                              ninfo("Tunnel established\n");
                              ws->state = WEBCLIENT_STATE_TUNNEL_ESTABLISHED;
                            }
                          else
                            {
                              ninfo("HTTP error from tunnelling proxy: %u\n",
                                    ctx->http_status);
                              ws->state = WEBCLIENT_STATE_DATA;
                            }
                        }
                      else
                        {
                          ws->state = WEBCLIENT_STATE_DATA;
                        }
                    }

                  goto exit;
                }

              /* Truncate the trailing \r\n */

              if (got_nl)
                {
                  ndx--;
                  if (ws->line[ndx] != ISO_CR)
                    {
                      nerr("ERROR: unexpected EOL from the server\n");
                      return -EPROTO;
                    }
                }

              ws->line[ndx] = '\0';

              if (!strchr(ws->line, ':'))
                {
                  if (got_nl)
                    {
                      nerr("ERROR: invalid header possibly due to "
                           "small CONFIG_WEBCLIENT_MAXHTTPLINE\n");
                      return -E2BIG;
                    }

                  nerr("ERROR: invalid header\n");
                  return -EPROTO;
                }

              if (ctx->header_callback)
                {
                  ret = ctx->header_callback(&ws->line[0], !got_nl,
                                             ctx->header_callback_arg);
                  if (ret != 0)
                    {
                      goto exit;
                    }
                }

              /* Check for specific HTTP header fields. */

#ifdef CONFIG_WEBCLIENT_GETMIMETYPE
              if (strncasecmp(ws->line, g_httpcontenttype,
                              strlen(g_httpcontenttype)) == 0)
                {
                  /* Found Content-type field. */

                  char *dest = strchr(ws->line, ';');
                  if (dest != NULL)
                    {
                      *dest = 0;
                    }

                  strlcpy(ws->mimetype, ws->line + strlen(g_httpcontenttype),
                          sizeof(ws->mimetype));
                  found = true;
                }
              else
#endif
              if (strncasecmp(ws->line, g_httplocation,
                              strlen(g_httplocation)) == 0)
                {
                  /* Parse the new host and filename from the URL.
                   */

                  ninfo("Redirect to location: '%s'\n",
                        ws->line + strlen(g_httplocation));
                  ret = parseurl(ws->line + strlen(g_httplocation),
                                 &ws->target, false);
                  if (ret != 0)
                    {
                      goto exit;
                    }

                  ninfo("New hostname='%s' filename='%s'\n",
                        ws->target.hostname, ws->target.filename);
                  ws->internal_flags |= WGET_FLAG_GOT_LOCATION;
                  found = true;
                }
              else if (strncasecmp(ws->line, g_httpcontsize,
                                   strlen(g_httpcontsize)) == 0)
                {
                  found = true;
                  if (got_nl)
                    {
                      ret = wget_parseint(ws->line + strlen(g_httpcontsize),
                                          &ws->expected_resp_body_len, 10);
                      if (ret != 0)
                        {
                          goto exit;
                        }

                      ws->internal_flags |=
                          WGET_FLAG_GOT_CONTENT_LENGTH;
                      ninfo("Content-Length %ju\n",
                            ws->expected_resp_body_len);
                    }
                }
              else if (strncasecmp(ws->line, g_httptransferencoding,
                                   strlen(g_httptransferencoding)) == 0)
                {
                  FAR const char *encodings =
                      ws->line + strlen(g_httptransferencoding);

                  if (strcasecmp(encodings, "chunked"))
                    {
                      nerr("unknown encodings: '%s'\n", encodings);
                      return -EPROTO;
                    }

                  ninfo("transfer encodings: '%s'\n", encodings);
                  ws->internal_flags |= WGET_FLAG_CHUNKED;
                }
            }

          if (found && !got_nl)
            {
              /* We found something we might care.
               * but we couldn't process it correctly.
               */

              nerr("ERROR: truncated a header due to "
                   "small CONFIG_WEBCLIENT_MAXHTTPLINE\n");
              return -E2BIG;
            }

          /* We're done parsing ws->line, so we reset the index.
           *
           * If we haven't seen the entire line yet (!got_nl),
           * skip the rest of the line.
           * Otherwise, start processing the next line.
           */

          ndx = 0;
          ws->skip_to_next_line = !got_nl;
        }
      else
        {
          ndx++;
        }
    }

exit:
  ws->offset = offset;
  ws->ndx    = ndx;
  return ret;
}

/****************************************************************************
 * Name: wget_parsechunkheader
 ****************************************************************************/

static inline int wget_parsechunkheader(struct webclient_context *ctx,
                                        struct wget_s *ws)
{
  int offset;
  int ndx;
  int ret = OK;

  UNUSED(ctx);

  offset = ws->offset;
  ndx    = ws->ndx;

  while (offset < ws->datend)
    {
      bool got_nl;

      ws->line[ndx] = ws->buffer[offset++];
      got_nl = ws->line[ndx] == ISO_NL;
      if (got_nl || ndx == CONFIG_WEBCLIENT_MAXHTTPLINE - 1)
        {
          bool found_extension = false;

          /* We have an entire header line in ws->line, or
           * our buffer is already full, so we start parsing it.
           */

          /* Should always be true */

          if (ndx > 0)
            {
              FAR char *semicolon;

              ninfo("Got chunk header line%s: %.*s\n",
                    got_nl ? "" : " (truncated)",
                    ndx - 1, &ws->line[0]);

              if (ws->line[0] == ISO_CR)
                {
                  nerr("ERROR: empty chunk header\n");
                  ret = -EPROTO;
                  break;
                }

              /* Truncate the trailing \r\n */

              if (got_nl)
                {
                  ndx--;
                  if (ws->line[ndx] != ISO_CR)
                    {
                      nerr("ERROR: unexpected EOL from the server\n");
                      ret = -EPROTO;
                      break;
                    }
                }

              ws->line[ndx] = '\0';

              semicolon = strchr(ws->line, ';');
              if (semicolon != NULL)
                {
                  found_extension = true;
                  ninfo("Ignoring extentions in chunk header\n");
                  *semicolon = 0;
                }
            }

          if (!got_nl && !found_extension)
            {
              /* We found something we might care.
               * but we couldn't process it correctly.
               */

              nerr("ERROR: truncated a header due to "
                   "small CONFIG_WEBCLIENT_MAXHTTPLINE\n");
              ret = -E2BIG;
              break;
            }

          ret = wget_parseint(ws->line, &ws->chunk_len, 16);
          if (ret != 0)
            {
              break;
            }

          if (ws->chunk_len != 0)
            {
              ninfo("Receiving a chunk with %ju bytes\n", ws->chunk_len);
              ws->state = WEBCLIENT_STATE_CHUNKED_DATA;
              ws->chunk_received = 0;
            }
          else
            {
              ws->state = WEBCLIENT_STATE_CHUNKED_TRAILER;
            }

          ndx = 0;
          break;
        }
      else
        {
          ndx++;
        }
    }

  ws->offset = offset;
  ws->ndx    = ndx;
  return ret;
}

/****************************************************************************
 * Name: wget_parsechunkenddata
 ****************************************************************************/

static inline int wget_parsechunkenddata(struct webclient_context *ctx,
                                         struct wget_s *ws)
{
  int offset;
  int ndx;
  int ret = OK;

  UNUSED(ctx);

  offset = ws->offset;
  ndx    = ws->ndx;

  while (offset < ws->datend)
    {
      ws->line[ndx] = ws->buffer[offset++];
      if (ws->line[ndx] == ISO_NL)
        {
          if (ndx == 0)
            {
              ret = -EPROTO;
              break;
            }

          if (ws->line[ndx - 1] != ISO_CR)
            {
              ret = -EPROTO;
              break;
            }

          if (ndx != 1)
            {
              ret = -EPROTO;
              break;
            }

          if (ws->chunk_len == 0)
            {
              ws->state = WEBCLIENT_STATE_CHUNKED_TRAILER;
            }
          else
            {
              ws->state = WEBCLIENT_STATE_CHUNKED_HEADER;
            }

          ndx = 0;
          break;
        }

      ndx++;
    }

  ws->offset = offset;
  ws->ndx    = ndx;
  return ret;
}

/****************************************************************************
 * Name: wget_parsechunktrailer
 ****************************************************************************/

static inline int wget_parsechunktrailer(struct webclient_context *ctx,
                                         struct wget_s *ws)
{
  int offset;
  int ndx;
  int ret = OK;

  UNUSED(ctx);

  offset = ws->offset;
  ndx    = ws->ndx;

  while (offset < ws->datend)
    {
      ws->line[ndx] = ws->buffer[offset++];
      if (ws->line[ndx] == ISO_NL)
        {
          if (ndx == 0)
            {
              ret = -EPROTO;
              break;
            }

          if (ws->line[ndx - 1] != ISO_CR)
            {
              ret = -EPROTO;
              break;
            }

          if (ndx != 1)
            {
              /* Ignore all non empty lines. */

              ndx = 0;
              continue;
            }

          ws->state = WEBCLIENT_STATE_WAIT_CLOSE;
          break;
        }

      ndx++;
    }

  ws->offset = offset;
  ws->ndx    = ndx;
  return ret;
}

/****************************************************************************
 * Name: wget_gethostip
 *
 * Description:
 *   Call getaddrinfo() to get the IPv4 address associated with a hostname.
 *
 * Input Parameters
 *   hostname - The host name to use in the nslookup.
 *
 * Output Parameters
 *   dest     - The location to return the IPv4 address.
 *
 * Returned Value:
 *   Zero (OK) on success; ERROR on failure.
 *
 ****************************************************************************/

static int wget_gethostip(FAR char *hostname, FAR struct in_addr *dest)
{
#ifdef CONFIG_LIBC_NETDB
  FAR struct addrinfo hint;
  FAR struct addrinfo *info;
  FAR struct sockaddr_in *addr;

  memset(&hint, 0, sizeof(hint));
  hint.ai_family = AF_INET;

  if (getaddrinfo(hostname, NULL, &hint, &info) != OK)
    {
      return ERROR;
    }

  addr = (FAR struct sockaddr_in *)info->ai_addr;
  memcpy(dest, &addr->sin_addr, sizeof(struct in_addr));

  freeaddrinfo(info);
  return OK;
#else
  /* No host name support */

  /* Convert strings to numeric IPv4 address */

  int ret = inet_pton(AF_INET, hostname, dest);

  /* The inet_pton() function returns 1 if the conversion succeeds. It will
   * return 0 if the input is not a valid IPv4 dotted-decimal string or -1
   * with errno set to EAFNOSUPPORT if the address family argument is
   * unsupported.
   */

  return (ret > 0) ? OK : ERROR;
#endif
}

/****************************************************************************
 * Public Functions
 ****************************************************************************/

/****************************************************************************
 * Name: webclient_conn_send
 ****************************************************************************/

ssize_t webclient_conn_send(FAR struct webclient_conn_s *conn,
                            FAR const void *buffer, size_t len)
{
  if (conn->tls)
    {
      return conn->tls_ops->send(conn->tls_ctx, conn->tls_conn, buffer, len);
    }

  while (true)
    {
      ssize_t ret = send(conn->sockfd, buffer, len, 0);
      if (ret == -1)
        {
          if (errno == EINTR)
            {
              continue;
            }

          if (errno == EAGAIN)
            {
              conn->flags |= CONN_WANT_WRITE;
            }

          return -errno;
        }

      return ret;
    }
}

/****************************************************************************
 * Name: webclient_conn_recv
 ****************************************************************************/

ssize_t webclient_conn_recv(FAR struct webclient_conn_s *conn,
                            FAR void *buffer, size_t len)
{
  if (conn->tls)
    {
      return conn->tls_ops->recv(conn->tls_ctx, conn->tls_conn, buffer, len);
    }

  while (true)
    {
      ssize_t ret = recv(conn->sockfd, buffer, len, 0);
      if (ret == -1)
        {
          if (errno == EINTR)
            {
              continue;
            }

          if (errno == EAGAIN)
            {
              conn->flags |= CONN_WANT_READ;
            }

          return -errno;
        }

      return ret;
    }
}

/****************************************************************************
 * Name: webclient_conn_close
 ****************************************************************************/

void webclient_conn_close(FAR struct webclient_conn_s *conn)
{
  if (conn->tls)
    {
      conn->tls_ops->close(conn->tls_ctx, conn->tls_conn);
    }
  else
    {
      close(conn->sockfd);
    }
}

/****************************************************************************
 * Name: webclient_conn_free
 ****************************************************************************/

void webclient_conn_free(FAR struct webclient_conn_s *conn)
{
  DEBUGASSERT(conn != NULL);
  free(conn);
}

/****************************************************************************
 * Name: webclient_perform
 *
 * Returned Value:
 *               0: if the operation completed successfully;
 *  Negative errno: On a failure
 *
 ****************************************************************************/

int webclient_perform(FAR struct webclient_context *ctx)
{
  struct wget_s *ws;
  struct timeval tv;
  char *dest;
  char *ep;
  struct webclient_conn_s *conn;
  FAR const struct webclient_tls_ops *tls_ops = ctx->tls_ops;
  FAR const char *method = ctx->method;
  FAR void *tls_ctx = ctx->tls_ctx;
  unsigned int i;
  int len;
  int ret;

#ifdef CONFIG_DEBUG_ASSERTIONS
  DEBUGASSERT(ctx->state == WEBCLIENT_CONTEXT_STATE_INITIALIZED ||
              (ctx->state == WEBCLIENT_CONTEXT_STATE_IN_PROGRESS &&
               (ctx->flags & WEBCLIENT_FLAG_NON_BLOCKING) != 0));
#endif

#if defined(CONFIG_WEBCLIENT_NET_LOCAL)
  if (ctx->unix_socket_path != NULL && ctx->proxy != NULL)
    {
      nerr("ERROR: proxy with AF_LOCAL is not implemented");
      _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
      return -ENOTSUP;
    }
#endif

  /* Initialize the state structure */

  if (ctx->ws == NULL)
    {
      ws = calloc(1, sizeof(struct wget_s));
      if (!ws)
        {
          _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
          return -errno;
        }

      ws->conn = calloc(1, sizeof(struct webclient_conn_s));
      if (!ws->conn)
        {
          free_ws(ws);
          _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
          return -errno;
        }

      ws->buffer = ctx->buffer;
      ws->buflen = ctx->buflen;

      /* Parse the hostname (with optional port number) and filename
       * from the URL.
       */

      if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) == 0)
        {
          ret = parseurl(ctx->url, &ws->target, false);
          if (ret != 0)
            {
              nwarn("WARNING: Malformed URL: %s\n", ctx->url);
              free_ws(ws);
              _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
              return ret;
            }
        }

      if (ctx->proxy != NULL)
        {
          /* Note: reject a proxy string w/o port number specified.
           * It's better to be explicit because the default number varies
           * among HTTP client implementations.
           * (80, 1080, 3128, 8080, ...)
           */

          ret = parseurl(ctx->proxy, &ws->proxy, true);
          if (ret != 0)
            {
              nerr("ERROR: Malformed proxy setting: %s\n", ctx->proxy);
              free_ws(ws);
              _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
              return ret;
            }

          if (strcmp(ws->proxy.scheme, "http") ||
              strcmp(ws->proxy.filename, "/"))
            {
              nerr("ERROR: Unsupported proxy setting: %s\n", ctx->proxy);
              free_ws(ws);
              _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
              return -ENOTSUP;
            }
        }

      ws->state = WEBCLIENT_STATE_SOCKET;
      ctx->ws = ws;
    }

  ws = ctx->ws;

  ninfo("hostname='%s' filename='%s'\n", ws->target.hostname,
        ws->target.filename);

  /* The following sequence may repeat indefinitely if we are redirected */

  conn = ws->conn;
  do
    {
      if (ws->state == WEBCLIENT_STATE_SOCKET)
        {
          if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) != 0)
            {
              conn->tls = false;
            }
          else if (!strcmp(ws->target.scheme, "https") && tls_ops != NULL)
            {
              conn->tls = true;
              conn->tls_ops = tls_ops;
              conn->tls_ctx = ctx->tls_ctx;
            }
          else if (!strcmp(ws->target.scheme, "http"))
            {
              conn->tls = false;
            }
          else
            {
              nerr("ERROR: unsupported scheme: %s\n", ws->target.scheme);
              free_ws(ws);
              _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
              return -ENOTSUP;
            }

          /* Re-initialize portions of the state structure that could have
           * been left from the previous time through the loop and should not
           * persist with the new connection.
           */

          ws->httpstatus = HTTPSTATUS_NONE;
          ws->offset     = 0;
          ws->datend     = 0;
          ws->ndx        = 0;
          ws->redirected = 0;

          if (conn->tls)
            {
#if defined(CONFIG_WEBCLIENT_NET_LOCAL)
              if (ctx->unix_socket_path != NULL)
                {
                  nerr("ERROR: TLS on AF_LOCAL socket is not implemented\n");
                  free_ws(ws);
                  _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
                  return -ENOTSUP;
                }
#endif

              if (ctx->proxy != NULL)
                {
                  FAR struct webclient_context *tunnel;

                  DEBUGASSERT(ws->tunnel == NULL);

                  if (tls_ops->init_connection == NULL)
                    {
                      nerr("ERROR: TLS over proxy is not implemented\n");
                      ret = -ENOTSUP;
                      goto errout_with_errno;
                    }

                  /* Create a temporary context to establish a tunnel. */

                  ws->tunnel = tunnel = calloc(1, sizeof(*ws->tunnel));
                  if (tunnel == NULL)
                    {
                      ret = -ENOMEM;
                      goto errout_with_errno;
                    }

                  webclient_set_defaults(tunnel);
                  tunnel->method = "CONNECT";
                  tunnel->flags |= WEBCLIENT_FLAG_TUNNEL;
                  tunnel->tunnel_target_host = ws->target.hostname;
                  tunnel->tunnel_target_port = ws->target.port;
                  tunnel->proxy = ctx->proxy;
                  tunnel->proxy_headers = ctx->proxy_headers;
                  tunnel->proxy_nheaders = ctx->proxy_nheaders;

                  /* Inherit some configurations.
                   *
                   * Revisit: should there be independent configurations?
                   */

                  tunnel->protocol_version = ctx->protocol_version;
                  if ((ctx->flags & WEBCLIENT_FLAG_NON_BLOCKING) != 0)
                    {
                      tunnel->flags |= WEBCLIENT_FLAG_NON_BLOCKING;
                    }

                  /* Abuse the buffer of the original request.
                   * It's safe with the current usage.
                   */

                  tunnel->buffer = ctx->buffer;
                  tunnel->buflen = ctx->buflen;
                }
            }
          else
            {
              int domain;

#if defined(CONFIG_WEBCLIENT_NET_LOCAL)
              if (ctx->unix_socket_path != NULL)
                {
                  domain = PF_LOCAL;
                }
              else
#endif
                {
                  domain = PF_INET;
                }

              /* Create a socket */

              conn->sockfd = socket(domain, SOCK_STREAM, 0);
              if (conn->sockfd < 0)
                {
                  ret = -errno;
                  nerr("ERROR: socket failed: %d\n", errno);
                  goto errout_with_errno;
                }

              ws->need_conn_close = true;

              if ((ctx->flags & WEBCLIENT_FLAG_NON_BLOCKING) != 0)
                {
                  int flags = fcntl(conn->sockfd, F_GETFL, 0);
                  ret = fcntl(conn->sockfd, F_SETFL, flags | O_NONBLOCK);
                  if (ret == -1)
                    {
                      ret = -errno;
                      nerr("ERROR: F_SETFL failed: %d\n", ret);
                      goto errout_with_errno;
                    }
                }
              else
                {
                  /* Set send and receive timeout values */

                  tv.tv_sec  = ctx->timeout_sec;
                  tv.tv_usec = 0;

                  /* Check return value one by one */

                  ret = setsockopt(conn->sockfd, SOL_SOCKET, SO_RCVTIMEO,
                                   &tv, sizeof(struct timeval));
                  if (ret != 0)
                    {
                      ret = -errno;
                      nerr("ERROR: setsockopt failed: %d\n", ret);
                      goto errout_with_errno;
                    }

                  ret = setsockopt(conn->sockfd, SOL_SOCKET, SO_SNDTIMEO,
                                   &tv, sizeof(struct timeval));
                  if (ret != 0)
                    {
                      ret = -errno;
                      nerr("ERROR: setsockopt failed: %d\n", ret);
                      goto errout_with_errno;
                    }
                }
            }

          ws->state = WEBCLIENT_STATE_CONNECT;
        }

      if (ws->state == WEBCLIENT_STATE_CONNECT)
        {
          if (ws->tunnel != NULL)
            {
              ret = webclient_perform(ws->tunnel);
              if (ret == 0)
                {
                  unsigned int http_status = ws->tunnel->http_status;

                  if (http_status / 100 != 2)
                    {
                      nerr("HTTP-level error %u on tunnel attempt\n",
                           http_status);

                      /* 407 Proxy Authentication Required */

                      if (http_status == 407)
                        {
                          ret = -EPERM;
                        }
                      else
                        {
                          ret = -EIO;
                        }
                    }
                }

              if (ret == 0)
                {
                  FAR struct webclient_conn_s *tunnel_conn;

                  webclient_get_tunnel(ws->tunnel, &tunnel_conn);
                  DEBUGASSERT(tunnel_conn != NULL);
                  DEBUGASSERT(!tunnel_conn->tls);
                  free(ws->tunnel);
                  ws->tunnel = NULL;

                  if (conn->tls)
                    {
                      /* Revisit: tunnel_conn here should have
                       * timeout configured already.
                       * Configuring it again here is redundant.
                       */

                      ret = tls_ops->init_connection(tls_ctx,
                                                     tunnel_conn,
                                                     ws->target.hostname,
                                                     ctx->timeout_sec,
                                                     &conn->tls_conn);
                      if (ret == 0)
                        {
                          /* Note: tunnel_conn has been consumed by
                           * tls_ops->init_connection
                           */

                          ws->need_conn_close = true;
                        }
                      else
                        {
                          /* Note: restarting tls_ops->init_connection
                           * is not implemented
                           */

                          DEBUGASSERT(ret != -EAGAIN &&
                                      ret != -EINPROGRESS &&
                                      ret != -EALREADY);
                          webclient_conn_close(tunnel_conn);
                          webclient_conn_free(tunnel_conn);
                        }
                    }
                  else
                    {
                      conn->sockfd = tunnel_conn->sockfd;
                      ws->need_conn_close = true;
                      webclient_conn_free(tunnel_conn);
                    }
                }
            }
          else if (conn->tls)
            {
              char port_str[sizeof("65535")];

#if defined(CONFIG_WEBCLIENT_NET_LOCAL)
              if (ctx->unix_socket_path != NULL)
                {
                  nerr("ERROR: TLS on AF_LOCAL socket is not implemented\n");
                  free_ws(ws);
                  _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
                  return -ENOTSUP;
                }
#endif

              snprintf(port_str, sizeof(port_str), "%u", ws->target.port);
              ret = tls_ops->connect(tls_ctx, ws->target.hostname, port_str,
                                     ctx->timeout_sec,
                                     &conn->tls_conn);
              if (ret == 0)
                {
                  ws->need_conn_close = true;
                }
            }
          else
            {
#if defined(CONFIG_WEBCLIENT_NET_LOCAL)
              struct sockaddr_un server_un;
#endif
              struct sockaddr_in server_in;
              const struct sockaddr *server_address;
              socklen_t server_address_len;

#if defined(CONFIG_WEBCLIENT_NET_LOCAL)
              if (ctx->unix_socket_path != NULL)
                {
                  memset(&server_un, 0, sizeof(server_un));
                  server_un.sun_family = AF_LOCAL;
                  strlcpy(server_un.sun_path, ctx->unix_socket_path,
                          sizeof(server_un.sun_path));
#if !defined(__NuttX__) && !defined(__linux__)
                  server_un.sun_len = SUN_LEN(&server_un);
#endif
                  server_address = (const struct sockaddr *)&server_un;
                  server_address_len = sizeof(server_un);
                }
              else
#endif
                {
                  /* Get the server address from the host name */

                  FAR struct wget_target_s *target;

                  if (ctx->proxy != NULL)
                    {
                      target = &ws->proxy;
                    }
                  else
                    {
                      target = &ws->target;
                    }

                  server_in.sin_family = AF_INET;
                  server_in.sin_port   = htons(target->port);
                  ret = wget_gethostip(target->hostname,
                                       &server_in.sin_addr);
                  if (ret < 0)
                    {
                      /* Could not resolve host (or malformed IP address) */

                      nwarn("WARNING: Failed to resolve hostname\n");
                      ret = -EHOSTUNREACH;
                      goto errout_with_errno;
                    }

                  server_address = (const struct sockaddr *)&server_in;
                  server_address_len = sizeof(struct sockaddr_in);
                }

              /* Connect to server.  First we have to set some fields in the
               * 'server' address structure.  The system will assign me an
               * arbitrary local port that is not in use.
               */

              while (true)
                {
                  ret = connect(conn->sockfd, server_address,
                                server_address_len);
                  if (ret == -1)
                    {
                      ret = -errno;
                      DEBUGASSERT(ret < 0);
                      if (ret == -EINTR)
                        {
                          continue;
                        }

                      if (ret == -EISCONN)
                        {
                          ret = 0;
                        }
                    }
                  break;
                }
            }

          if (ret < 0)
            {
              nerr("ERROR: connect failed: %d\n", errno);
              goto errout_with_errno;
            }

          ws->state = WEBCLIENT_STATE_PREPARE_REQUEST;
        }

      if (ws->state == WEBCLIENT_STATE_PREPARE_REQUEST)
        {
          /* Send the request */

          dest = ws->buffer;
          ep = ws->buffer + ws->buflen;
          dest = append(dest, ep, method);
          dest = append(dest, ep, " ");

          if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) != 0)
            {
              /* Use authority-form for a tunnel
               *
               * https://datatracker.ietf.org/doc/html/rfc7231#section-4.3.6
               * https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3
               */

              char port_str[sizeof("65535")];

              dest = append(dest, ep, ctx->tunnel_target_host);
              dest = append(dest, ep, ":");
              snprintf(port_str, sizeof(port_str), "%u",
                       ctx->tunnel_target_port);
              dest = append(dest, ep, port_str);
            }
          else if (ctx->proxy != NULL)
            {
              /* Use absolute-form for a proxy
               *
               * https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.2
               */

              char port_str[sizeof("65535")];

              dest = append(dest, ep, ws->target.scheme);
              dest = append(dest, ep, "://");
              dest = append(dest, ep, ws->target.hostname);
              dest = append(dest, ep, ":");
              snprintf(port_str, sizeof(port_str), "%u", ws->target.port);
              dest = append(dest, ep, port_str);
              dest = append(dest, ep, ws->target.filename);
            }
          else
            {
              /* Otherwise, use origin-form */

#ifndef WGET_USE_URLENCODE
              dest = append(dest, ep, ws->target.filename);
#else
              /* TODO: should we use wget_urlencode_strcpy? */

              dest = append(dest, ep, ws->target.filename);
#endif
            }

          dest = append(dest, ep, " ");
          if (ctx->protocol_version == WEBCLIENT_PROTOCOL_VERSION_HTTP_1_0)
            {
              dest = append(dest, ep, g_http10);
            }
          else if (ctx->protocol_version ==
                   WEBCLIENT_PROTOCOL_VERSION_HTTP_1_1)
            {
              dest = append(dest, ep, g_http11);
            }
          else
            {
              ret = -EINVAL;
              goto errout_with_errno;
            }

          dest = append(dest, ep, g_httpcrnl);

          /* Note about proxy and Host header:
           *
           * https://datatracker.ietf.org/doc/html/rfc7230#section-5.4
           * > A client MUST send a Host header field in an HTTP/1.1
           * > request even if the request-target is in the absolute-form,
           * > since this allows the Host information to be forwarded
           * > through ancient HTTP/1.0 proxies that might not have
           * > implemented Host.
           *
           * > When a proxy receives a request with an absolute-form of
           * > request-target, the proxy MUST ignore the received Host
           * > header field (if any) and instead replace it with the host
           * > information of the request-target.
           */

          dest = append(dest, ep, g_httphost);
          dest = append(dest, ep, ws->target.hostname);
          dest = append(dest, ep, g_httpcrnl);

          /* Append user-specified headers.
           *
           * - For non-proxied request,
           *   only send "headers".
           *
           * - For proxied request, (traditional http proxy)
           *   send both of "headers" and "proxy_headers".
           *
           * - For tunneling request, (WEBCLIENT_FLAG_TUNNEL)
           *   only send "proxy_headers".
           */

          if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) == 0)
            {
              for (i = 0; i < ctx->nheaders; i++)
                {
                  dest = append(dest, ep, ctx->headers[i]);
                  dest = append(dest, ep, g_httpcrnl);
                }
            }

          if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) != 0 ||
              ctx->proxy != NULL)
            {
              for (i = 0; i < ctx->proxy_nheaders; i++)
                {
                  dest = append(dest, ep, ctx->proxy_headers[i]);
                  dest = append(dest, ep, g_httpcrnl);
                }
            }

          if (ctx->bodylen)
            {
              char post_size[sizeof("18446744073709551615")];

              dest = append(dest, ep, g_httpcontsize);
              snprintf(post_size, sizeof(post_size), "%zu", ctx->bodylen);
              dest = append(dest, ep, post_size);
              dest = append(dest, ep, g_httpcrnl);
            }

          if (ctx->protocol_version == WEBCLIENT_PROTOCOL_VERSION_HTTP_1_1)
            {
              /* We don't implement persistect connections. */

              dest = append(dest, ep, g_httpconn_close);
              dest = append(dest, ep, g_httpcrnl);
            }

          dest = append(dest, ep, g_httpuseragentfields);

          if (dest == NULL)
            {
              ret = -E2BIG;
              goto errout_with_errno;
            }

          len = dest - ws->buffer;

          ws->state = WEBCLIENT_STATE_SEND_REQUEST;
          ws->state_offset = 0;
          ws->state_len = len;
        }

      if (ws->state == WEBCLIENT_STATE_SEND_REQUEST)
        {
          ssize_t ssz;

          ssz = webclient_conn_send(conn,
                                    ws->buffer + ws->state_offset,
                                    ws->state_len);
          if (ssz < 0)
            {
              ret = ssz;
              nerr("ERROR: send failed: %d\n", -ret);
              goto errout_with_errno;
            }

          if (ssz >= 0)
            {
              ws->state_offset += ssz;
              ws->state_len -= ssz;
              if (ws->state_len == 0)
                {
                  ws->state = WEBCLIENT_STATE_SEND_REQUEST_BODY;
                  ws->state_offset = 0;
                  ws->state_len = ctx->bodylen;
                  ws->data_buffer = NULL;
                  ninfo("Sending %zu bytes request body\n", ws->state_len);
                }
            }
        }

      if (ws->state == WEBCLIENT_STATE_SEND_REQUEST_BODY)
        {
          /* In this state,
           *
           * ws->data_buffer  the data given by the user
           * ws->data_offset  the byte offset in the entire body
           * ws->data_len     the byte size of the data in ws->data_buffer
           * ws->state_offset the send offset in ws->data_buffer
           * ws->state_len    the number of remaining bytes to send (total)
           */

          if (ws->state_len == 0)
            {
              ninfo("Finished sending request body\n");
              ws->state = WEBCLIENT_STATE_STATUSLINE;
            }
          else if (ws->data_buffer == NULL)
            {
              FAR const void *input_buffer;
              size_t input_buffer_size = ws->buflen;

              size_t todo = ws->state_len;
              if (input_buffer_size > todo)
                {
                  input_buffer_size = todo;
                }

              input_buffer = ws->buffer;
              ret = ctx->body_callback(ws->buffer,
                                       &input_buffer_size,
                                       &input_buffer,
                                       todo,
                                       ctx->body_callback_arg);
              if (ret < 0)
                {
                  nerr("ERROR: body_callback failed: %d\n", -ret);
                  goto errout_with_errno;
                }

              ninfo("Got %zu bytes body chunk / total remaining %zu bytes\n",
                    input_buffer_size, ws->state_len);
              ws->data_buffer = input_buffer;
              ws->data_len = input_buffer_size;
              ws->state_offset = 0;
            }
          else
            {
              size_t bytes_to_send = ws->data_len - ws->state_offset;

              DEBUGASSERT(bytes_to_send <= ws->state_len);
              ssize_t ssz = webclient_conn_send(conn,
                                                (char *)ws->data_buffer +
                                                ws->state_offset,
                                                bytes_to_send);
              if (ssz < 0)
                {
                  ret = ssz;
                  nerr("ERROR: send failed: %d\n", -ret);
                  goto errout_with_errno;
                }

              ninfo("Sent %zd bytes request body at offset %ju "
                    "in the %zu bytes chunk, "
                    "total remaining %zu bytes\n",
                    ssz,
                    (uintmax_t)ws->state_offset,
                    ws->data_len,
                    ws->state_len);
              ws->state_len -= ssz;
              ws->state_offset += ssz;
              DEBUGASSERT((size_t)ws->state_offset <= ws->data_len);
              if ((size_t)ws->state_offset == ws->data_len)
                {
                  ws->data_buffer = NULL;
                }
            }
        }

      /* Now loop to get the file sent in response to the GET.  This
       * loop continues until either we read the end of file (nbytes == 0)
       * or until we detect that we have been redirected.
       */

      if (ws->state == WEBCLIENT_STATE_STATUSLINE ||
          ws->state == WEBCLIENT_STATE_HEADERS ||
          ws->state == WEBCLIENT_STATE_DATA ||
          ws->state == WEBCLIENT_STATE_CHUNKED_HEADER ||
          ws->state == WEBCLIENT_STATE_CHUNKED_DATA)
        {
          for (; ; )
            {
              if (ws->datend - ws->offset == 0)
                {
                  size_t want = ws->buflen;
                  ssize_t ssz;

                  ninfo("Reading new data\n");
                  if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) != 0)
                    {
                      /* When tunnelling, we want to avoid troubles
                       * with reading the starting payload of the tunnelled
                       * protocol here, in case it's a server-speaks-first
                       * protocol.
                       */

                      want = 1;
                    }

                  ssz = webclient_conn_recv(conn, ws->buffer, want);
                  if (ssz < 0)
                    {
                      ret = ssz;
                      nerr("ERROR: recv failed: %d\n", -ret);
                      goto errout_with_errno;
                    }
                  else if (ssz == 0)
                    {
                      if (ws->state != WEBCLIENT_STATE_DATA &&
                          ws->state != WEBCLIENT_STATE_WAIT_CLOSE)
                        {
                          nerr("Connection lost unexpectedly\n");
                          ret = -ECONNABORTED;
                          goto errout_with_errno;
                        }

                      if ((ws->internal_flags &
                           WGET_FLAG_GOT_CONTENT_LENGTH) != 0 &&
                          ws->expected_resp_body_len !=
                          ws->received_body_len)
                        {
                          nerr("Unexpected response body length: "
                               "%ju != %ju\n",
                               ws->expected_resp_body_len,
                               ws->received_body_len);
                          ret = -EPROTO;
                          goto errout_with_errno;
                        }

                      ninfo("Connection lost\n");
                      ws->state = WEBCLIENT_STATE_CLOSE;
                      ws->redirected = 0;
                      break;
                    }

                  ninfo("Got %zd bytes data\n", ssz);
                  ws->offset = 0;
                  ws->datend = ssz;
                }

              /* Handle initial parsing of the status line */

              if (ws->state == WEBCLIENT_STATE_STATUSLINE)
                {
                  ret = wget_parsestatus(ctx, ws);
                  if (ret < 0)
                    {
                      goto errout_with_errno;
                    }
                }

              /* Parse the HTTP data */

              if (ws->state == WEBCLIENT_STATE_HEADERS)
                {
                  ret = wget_parseheaders(ctx, ws);
                  if (ret < 0)
                    {
                      goto errout_with_errno;
                    }
                }

              /* Parse the chunk header */

              if (ws->state == WEBCLIENT_STATE_CHUNKED_HEADER)
                {
                  ret = wget_parsechunkheader(ctx, ws);
                  if (ret < 0)
                    {
                      goto errout_with_errno;
                    }
                }

              if (ws->state == WEBCLIENT_STATE_CHUNKED_ENDDATA)
                {
                  ret = wget_parsechunkenddata(ctx, ws);
                  if (ret < 0)
                    {
                      goto errout_with_errno;
                    }
                }

              if (ws->state == WEBCLIENT_STATE_CHUNKED_TRAILER)
                {
                  ret = wget_parsechunktrailer(ctx, ws);
                  if (ret < 0)
                    {
                      goto errout_with_errno;
                    }
                }

              if (ws->state == WEBCLIENT_STATE_WAIT_CLOSE)
                {
                  uintmax_t received = ws->datend - ws->offset;
                  if (received != 0)
                    {
                      nerr("Unexpected %ju bytes data received", received);
                      ret = -EPROTO;
                      goto errout_with_errno;
                    }
                }

              /* Dispose of the data payload */

              if (ws->state == WEBCLIENT_STATE_DATA ||
                  ws->state == WEBCLIENT_STATE_CHUNKED_DATA)
                {
                  if (ws->httpstatus != HTTPSTATUS_MOVED)
                    {
                      uintmax_t received = ws->datend - ws->offset;
                      FAR char *orig_buffer = ws->buffer;
                      int orig_buflen = ws->buflen;

                      if ((ws->internal_flags & WGET_FLAG_GOT_LOCATION) != 0)
                        {
                          nwarn("WARNING: Unexpected Location header\n");
                        }

                      if (ws->state == WEBCLIENT_STATE_CHUNKED_DATA)
                        {
                          uintmax_t chunk_left =
                              ws->chunk_len - ws->chunk_received;

                          if (received > chunk_left)
                            {
                              received = chunk_left;
                            }

                          ws->chunk_received += received;
                        }

                      ninfo("Processing resp body %ju - %ju\n",
                            ws->received_body_len,
                            ws->received_body_len + received);
                      ws->received_body_len += received;

                      /* Let the client decide what to do with the
                       * received file.
                       */

                      if (received == 0)
                        {
                          /* We don't have data to give to the client yet. */
                        }
                      else if (ctx->sink_callback)
                        {
                          ret = ctx->sink_callback(&ws->buffer, ws->offset,
                                                   ws->offset + received,
                                                   &ws->buflen,
                                                   ctx->sink_callback_arg);
                          if (ret != 0)
                            {
                              goto errout_with_errno;
                            }
                        }
                      else if (ctx->callback)
                        {
                          ctx->callback(&ws->buffer, ws->offset,
                                        ws->offset + received,
                                        &ws->buflen, ctx->sink_callback_arg);
                        }

                      ws->offset += received;

                      /* The buffer swapping API doesn't work for
                       * HTTP 1.1 chunked transfer because the buffer here
                       * might already contain the next chunk header.
                       */

                      if (ctx->protocol_version ==
                          WEBCLIENT_PROTOCOL_VERSION_HTTP_1_1)
                        {
                          if (orig_buffer != ws->buffer ||
                              orig_buflen != ws->buflen)
                            {
                              ret = -EINVAL;
                              goto errout_with_errno;
                            }
                        }

                      if (ws->state == WEBCLIENT_STATE_CHUNKED_DATA)
                        {
                          if (ws->chunk_len == ws->chunk_received)
                            {
                              ws->state = WEBCLIENT_STATE_CHUNKED_ENDDATA;
                              ws->ndx = 0;
                            }
                        }
                    }
                  else
                    {
                      if ((ws->internal_flags & WGET_FLAG_GOT_LOCATION) == 0)
                        {
                          nerr("ERROR: Redirect w/o Location header\n");
                          ret = -EPROTO;
                          goto errout_with_errno;
                        }

                      ws->nredirect++;
                      if (ws->nredirect > CONFIG_WEBCLIENT_MAX_REDIRECT)
                        {
                          nerr("ERROR: too many redirects (%u)\n",
                               ws->nredirect);
                          ret = -ELOOP;
                          goto errout_with_errno;
                        }

                      ws->state = WEBCLIENT_STATE_CLOSE;
                      ws->redirected = 1;
                      break;
                    }
                }

              if (ws->state == WEBCLIENT_STATE_TUNNEL_ESTABLISHED)
                {
                  break;
                }
            }
        }

      if (ws->state == WEBCLIENT_STATE_CLOSE)
        {
          webclient_conn_close(conn);
          ws->need_conn_close = false;
          if (ws->redirected)
            {
              ws->state = WEBCLIENT_STATE_SOCKET;
            }
          else
            {
              ws->state = WEBCLIENT_STATE_DONE;
            }
        }
    }
  while (ws->state != WEBCLIENT_STATE_DONE &&
         ws->state != WEBCLIENT_STATE_TUNNEL_ESTABLISHED);

  if (ws->state == WEBCLIENT_STATE_DONE)
    {
      free_ws(ws);
      _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
    }
  else
    {
      _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_TUNNEL_ESTABLISHED);
    }

  return OK;

errout_with_errno:
  if ((ctx->flags & WEBCLIENT_FLAG_NON_BLOCKING) != 0 &&
      (ret == -EAGAIN || ret == -EINPROGRESS || ret == -EALREADY))
    {
      if (ret == -EINPROGRESS || ret == -EALREADY)
        {
          conn->flags |= CONN_WANT_WRITE;
        }

      _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_IN_PROGRESS);
      return -EAGAIN;
    }

  if (ws->need_conn_close)
    {
      webclient_conn_close(conn);
    }

  free_ws(ws);
  _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
  return ret;
}

/****************************************************************************
 * Name: webclient_abort
 *
 * Description:
 *  This function is used to free the resources used by webclient_perform()
 *  in case of non blocking I/O.
 *
 *  After webclient_perform() returned -EAGAIN, the application can either
 *  retry the operation by calling webclient_perform() again or abort
 *  the operation by calling this function.
 *
 ****************************************************************************/

void webclient_abort(FAR struct webclient_context *ctx)
{
  _CHECK_STATE(ctx, WEBCLIENT_CONTEXT_STATE_IN_PROGRESS);
  DEBUGASSERT((ctx->flags & WEBCLIENT_FLAG_NON_BLOCKING) != 0);

  struct wget_s *ws = ctx->ws;

  if (ws == NULL)
    {
      return;
    }

  if (ws->need_conn_close)
    {
      struct webclient_conn_s *conn = ws->conn;

      webclient_conn_close(conn);
    }

  if (ws->tunnel != NULL)
    {
      webclient_abort(ws->tunnel);
    }

  free_ws(ws);
  _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_ABORTED);
}

/****************************************************************************
 * Name: web_post_str
 ****************************************************************************/

#ifdef WGET_USE_URLENCODE
char *web_post_str(FAR char *buffer, int *size, FAR char *name,
                   FAR char *value)
{
  char *dst = buffer;
  buffer = wget_strcpy(buffer, name);
  buffer = wget_strcpy(buffer, "=");
  buffer = wget_urlencode_strcpy(buffer, value);
  *size  = buffer - dst;
  return dst;
}
#endif

/****************************************************************************
 * Name: web_post_strlen
 ****************************************************************************/

#ifdef WGET_USE_URLENCODE
int web_post_strlen(FAR char *name, FAR char *value)
{
  return strlen(name) + urlencode_len(value, strlen(value)) + 1;
}
#endif

/****************************************************************************
 * Name: web_posts_str
 ****************************************************************************/

#ifdef WGET_USE_URLENCODE
char *web_posts_str(FAR char *buffer, int *size, FAR char **name,
                    FAR char **value, int len)
{
  char *dst = buffer;
  int wlen;
  int i;

  for (i = 0; i < len; i++)
    {
      if (i > 0)
        {
          buffer = wget_strcpy(buffer, "&");
        }

      wlen    = *size;
      buffer  = web_post_str(buffer, &wlen, name[i], value[i]);
      buffer += wlen;
    }

  *size = buffer - dst;
  return dst;
}
#endif

/****************************************************************************
 * Name: web_posts_strlen
 ****************************************************************************/

#ifdef WGET_USE_URLENCODE
int web_posts_strlen(FAR char **name, FAR char **value, int len)
{
  int wlen = 0;
  int i;

  for (i = 0; i < len; i++)
    {
      wlen += web_post_strlen(name[i], value[i]);
    }

  return wlen + len - 1;
}
#endif

/****************************************************************************
 * Name: wget
 *
 * Description:
 *   Obtain the requested file from an HTTP server using the GET method.
 *
 * Input Parameters
 *   url      - A pointer to a string containing either the full URL to
 *              the file to get (e.g., http://www.nutt.org/index.html, or
 *              http://192.168.23.1:80/index.html).
 *   buffer   - A user provided buffer to receive the file data (also
 *              used for the outgoing GET request
 *   buflen   - The size of the user provided buffer
 *   callback - As data is obtained from the host, this function is
 *              to dispose of each block of file data as it is received.
 *   arg      - User argument passed to callback.
 *
 * Returned Value:
 *   0: if the GET operation completed successfully;
 *  -1: On a failure with errno set appropriately
 *
 ****************************************************************************/

int wget(FAR const char *url, FAR char *buffer, int buflen,
         wget_callback_t callback, FAR void *arg)
{
  struct webclient_context ctx;
  int ret;
  webclient_set_defaults(&ctx);
  ctx.method = "GET";
  ctx.url = url;
  ctx.buffer = buffer;
  ctx.buflen = buflen;
  ctx.callback = callback;
  ctx.sink_callback_arg = arg;
  ret = webclient_perform(&ctx);
  if (ret < 0)
    {
      errno = -ret;
      return ERROR;
    }

  return OK;
}

/****************************************************************************
 * Name: wget_post
 ****************************************************************************/

int wget_post(FAR const char *url, FAR const char *posts, FAR char *buffer,
              int buflen, wget_callback_t callback, FAR void *arg)
{
  static const char *headers = g_httpform;
  struct webclient_context ctx;
  int ret;
  webclient_set_defaults(&ctx);
  ctx.method = "POST";
  ctx.url = url;
  ctx.buffer = buffer;
  ctx.buflen = buflen;
  ctx.callback = callback;
  ctx.sink_callback_arg = arg;
  ctx.headers = &headers;
  ctx.nheaders = 1;
  webclient_set_static_body(&ctx, posts, strlen(posts));
  ret = webclient_perform(&ctx);
  if (ret < 0)
    {
      errno = -ret;
      return ERROR;
    }

  return OK;
}

/****************************************************************************
 * Name: webclient_set_defaults
 ****************************************************************************/

void webclient_set_defaults(FAR struct webclient_context *ctx)
{
  memset(ctx, 0, sizeof(*ctx));
  ctx->protocol_version = WEBCLIENT_PROTOCOL_VERSION_HTTP_1_0;
  ctx->method = "GET";
  ctx->timeout_sec = CONFIG_WEBCLIENT_TIMEOUT;
  _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_INITIALIZED);
}

/****************************************************************************
 * Name: webclient_set_static_body
 ****************************************************************************/

void webclient_set_static_body(FAR struct webclient_context *ctx,
                               FAR const void *body,
                               size_t bodylen)
{
  _CHECK_STATE(ctx, WEBCLIENT_CONTEXT_STATE_INITIALIZED);

  ctx->body_callback = webclient_static_body_func;
  ctx->body_callback_arg = (void *)body; /* discard const */
  ctx->bodylen = bodylen;
}

/****************************************************************************
 * Name: webclient_get_poll_info
 *
 * Description:
 *   This function retrieves the information necessary
 *   to wait for events when using non blocking I/O.
 *
 *   When using WEBCLIENT_FLAG_NON_BLOCKING, webclient_perform() can
 *   return -EAGAIN when it would otherwise block for I/O.
 *   In that case, the application can use this function to
 *   get the information necessary to wait for the I/O events
 *   using poll()/select(), by populating the the given
 *   webclient_poll_info structure with the information.
 *
 *   The following is an example to use this function to handle EAGAIN.
 *
 *        retry:
 *          ret = webclient_perform(&ctx);
 *          if (ret == -EAGAIN)
 *            {
 *              struct webclient_poll_info info;
 *              struct pollfd pfd;
 *
 *              ret = webclient_get_poll_info(&ctx, &info);
 *              if (ret != 0)
 *                {
 *                  ...
 *                }
 *
 *              memset(&pfd, 0, sizeof(pfd));
 *              pfd.fd = info.fd;
 *              if ((info.flags & WEBCLIENT_POLL_INFO_WANT_READ) != 0)
 *                {
 *                  pfd.events |= POLLIN;
 *                }
 *
 *              if ((info.flags & WEBCLIENT_POLL_INFO_WANT_WRITE) != 0)
 *                {
 *                  pfd.events |= POLLOUT;
 *                }
 *
 *              ret = poll(&pfd, 1, -1);
 *              if (ret != 0)
 *                {
 *                  ...
 *                }
 *
 *              goto retry;
 *            }
 *
 ****************************************************************************/

int webclient_get_poll_info(FAR struct webclient_context *ctx,
                             FAR struct webclient_poll_info *info)
{
  struct wget_s *ws;
  struct webclient_conn_s *conn;

  _CHECK_STATE(ctx, WEBCLIENT_CONTEXT_STATE_IN_PROGRESS);
  DEBUGASSERT((ctx->flags & WEBCLIENT_FLAG_NON_BLOCKING) != 0);

  ws = ctx->ws;
  if (ws == NULL)
    {
      return -EINVAL;
    }

  if (ws->tunnel != NULL)
    {
      return webclient_get_poll_info(ws->tunnel, info);
    }

  conn = ws->conn;
  if (conn->tls)
    {
      return ctx->tls_ops->get_poll_info(ctx->tls_ctx, conn->tls_conn, info);
    }

  info->fd = conn->sockfd;
  info->flags = conn->flags & (CONN_WANT_READ | CONN_WANT_WRITE);
  conn->flags &= ~(CONN_WANT_READ | CONN_WANT_WRITE);
  return 0;
}

/****************************************************************************
 * Name: webclient_get_tunnel
 *
 * Description:
 *   This function is used to get the webclient_conn_s, which describes
 *   the tunneled connection.
 *
 *   This function should be used exactly once after a successful
 *   call of webclient_perform with WEBCLIENT_FLAG_TUNNEL, with
 *   http_status 2xx.
 *
 *   This function also disposes the given webclient_context.
 *   The context will be invalid after a call of this function.
 *
 *   It's the caller's responsibility to close the returned
 *   webclient_conn_s, either using webclient_conn_close() or
 *   using the internal knowledge about the structure. (E.g.
 *   It's sometimes convenient/efficient for the caller to
 *   only keep conn->sockfd descriptor and free the rest of the
 *   structure using webclient_conn_free(). In that case, it will
 *   need to close() the descriptor after finishing on it.)
 *
 *   It's the caller's responsibility to free the returned
 *   webclient_conn_s using webclient_conn_free().
 *
 ****************************************************************************/

void webclient_get_tunnel(FAR struct webclient_context *ctx,
                          FAR struct webclient_conn_s **connp)
{
  struct wget_s *ws;
  struct webclient_conn_s *conn;

  _CHECK_STATE(ctx, WEBCLIENT_CONTEXT_STATE_TUNNEL_ESTABLISHED);
  DEBUGASSERT(ctx->http_status / 100 == 2);
  ws = ctx->ws;
  DEBUGASSERT(ws != NULL);
  conn = ws->conn;
  DEBUGASSERT(conn != NULL);
  *connp = conn;
  ws->conn = NULL;
  free_ws(ws);
  _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE);
}