| Search.setIndex({"docnames": ["_api/airflow/providers/apache/hdfs/hooks/hdfs/index", "_api/airflow/providers/apache/hdfs/hooks/index", "_api/airflow/providers/apache/hdfs/hooks/webhdfs/index", "_api/airflow/providers/apache/hdfs/index", "_api/airflow/providers/apache/hdfs/log/hdfs_task_handler/index", "_api/airflow/providers/apache/hdfs/log/index", "_api/airflow/providers/apache/hdfs/sensors/hdfs/index", "_api/airflow/providers/apache/hdfs/sensors/index", "_api/airflow/providers/apache/hdfs/sensors/web_hdfs/index", "changelog", "commits", "connections", "index", "installing-providers-from-sources", "logging/hdfs-task-handler", "logging/index", "operators/index", "operators/webhdfs", "security"], "filenames": ["_api/airflow/providers/apache/hdfs/hooks/hdfs/index.rst", "_api/airflow/providers/apache/hdfs/hooks/index.rst", "_api/airflow/providers/apache/hdfs/hooks/webhdfs/index.rst", "_api/airflow/providers/apache/hdfs/index.rst", "_api/airflow/providers/apache/hdfs/log/hdfs_task_handler/index.rst", "_api/airflow/providers/apache/hdfs/log/index.rst", "_api/airflow/providers/apache/hdfs/sensors/hdfs/index.rst", "_api/airflow/providers/apache/hdfs/sensors/index.rst", "_api/airflow/providers/apache/hdfs/sensors/web_hdfs/index.rst", "changelog.rst", "commits.rst", "connections.rst", "index.rst", "installing-providers-from-sources.rst", "logging/hdfs-task-handler.rst", "logging/index.rst", "operators/index.rst", "operators/webhdfs.rst", "security.rst"], "titles": ["<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.hdfs.hooks.hdfs</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.hdfs.hooks</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.hdfs.hooks.webhdfs</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.hdfs</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.hdfs.log.hdfs_task_handler</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.hdfs.log</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.hdfs.sensors.hdfs</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.hdfs.sensors</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">airflow.providers.apache.hdfs.sensors.web_hdfs</span></code>", "Changelog", "Package apache-airflow-providers-apache-hdfs", "Apache HDFS Connection", "<code class=\"docutils literal notranslate\"><span class=\"pre\">apache-airflow-providers-apache-hdfs</span></code>", "Installing from sources", "Writing logs to HDFS", "Logging for Tasks", "Apache Hadoop HDFS Operators", "WebHDFS Operators", "Releasing security patches"], "terms": {"except": [0, 2, 4, 9, 10, 18], "hdfshookexcept": 0, "arg": [0, 6], "kwarg": [0, 2, 4, 6, 8], "sourc": [0, 2, 3, 4, 6, 8], "base": [0, 2, 4, 6, 8, 9, 10], "airflowexcept": [0, 2, 9, 10], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18], "ha": [0, 4, 6, 9, 10], "been": [0, 4, 6, 9], "remov": [0, 6, 9, 10, 13], "i": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18], "function": [0, 4, 6, 10], "pleas": [0, 6, 13], "convert": [0, 6], "your": [0, 6, 9], "dag": [0, 6], "us": [0, 2, 4, 6, 9, 10, 11, 13, 14, 17, 18], "webhdfshook": [0, 2, 4, 9, 10, 14], "downgrad": [0, 6, 9], "below": [0, 6, 12, 13], "4": [0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14, 15, 16, 17, 18], "you": [0, 6, 9, 12, 13, 14, 18], "want": [0, 6, 9, 13, 18], "continu": [0, 6, 9], "If": [0, 2, 6, 9, 11, 13, 14], "earlier": [0, 6], "can": [0, 6, 9, 11, 12, 13, 14, 18], "latest": [0, 6, 9, 10, 18], "releas": [0, 6, 9, 10, 12], "3": [0, 6], "version": [0, 6, 9, 10, 12, 13, 18], "pip": [0, 6, 10, 12, 13], "instal": [0, 6, 9, 10, 18], "2": [0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14, 15, 16, 17, 18], "1": [0, 2, 6], "constraint": [0, 6], "hdfshook": [0, 9, 10], "basehook": [0, 2], "0": [0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14, 15, 16, 17, 18], "10": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18], "dev0": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18], "an": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18], "experiment": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18], "featur": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18], "webhdf": [1, 3, 9, 10, 12, 16], "web": [2, 11, 17], "log": [2, 3, 9, 10], "airflowwebhdfshookexcept": 2, "specif": [2, 10], "webhdfs_conn_id": [2, 8, 11], "default_conn_nam": 2, "proxy_us": 2, "none": [2, 4, 13], "interact": 2, "wrapper": 2, "around": 2, "hdfscli": 2, "librari": [2, 9], "paramet": [2, 4, 10, 11, 17], "str": [2, 8], "The": [2, 9, 11, 12, 13, 17, 18], "connect": [2, 9, 10, 14], "id": [2, 13, 14], "client": [2, 9], "user": [2, 11, 13, 14], "authent": [2, 9, 10], "conn_typ": 2, "conn_name_attr": 2, "webhdfs_default": [2, 8, 11, 14], "hook_nam": 2, "get_conn": 2, "establish": 2, "depend": [2, 9, 10, 18], "secur": [2, 17], "mode": 2, "set": [2, 4, 9, 10, 11], "via": [2, 12, 13], "config": 2, "environ": [2, 9], "variabl": [2, 9], "return": [2, 4], "insecurecli": 2, "kerberoscli": 2, "object": [2, 4], "type": [2, 9, 10, 11], "ani": [2, 13], "check_for_path": 2, "hdfs_path": 2, "check": [2, 10, 13, 17], "exist": [2, 12, 14], "path": [2, 14], "queri": 2, "filestatu": 2, "true": [2, 14], "fals": [2, 4, 11], "bool": 2, "load_fil": 2, "destin": 2, "overwrit": 2, "parallel": [2, 17], "upload": [2, 4], "file": [2, 4, 8, 9, 10, 12, 13], "local": [2, 4, 11, 13], "folder": [2, 8, 10, 13], "": [2, 9, 10, 13], "all": [2, 9, 10, 12, 18], "insid": [2, 10], "note": [2, 10], "impli": 2, "empti": 2, "creat": [2, 10, 13], "remot": 2, "ptarget": 2, "alreadi": [2, 9, 13], "directori": [2, 13], "int": 2, "number": [2, 10], "thread": 2, "A": 2, "valu": [2, 11], "neg": 2, "mani": [2, 9], "ar": [2, 9, 12, 13, 18], "keyword": [2, 10], "argument": [2, 10], "forward": 2, "read_fil": 2, "filenam": 2, "read": [2, 4, 9, 14], "from": [2, 4, 9, 10, 18], "raw": 2, "string": [2, 10], "byte": 2, "hook": [3, 4, 9, 10, 11], "hdfs_task_handl": [3, 5], "sensor": [3, 9, 10], "web_hdf": [3, 7], "__version__": [3, 10], "hdfstaskhandl": [4, 10], "base_log_fold": 4, "hdfs_log_fold": 4, "filename_templ": 4, "util": 4, "file_task_handl": 4, "filetaskhandl": 4, "logging_mixin": 4, "loggingmixin": 4, "handler": [4, 9, 10], "set_context": 4, "ti": 4, "task_inst": 4, "context": [4, 8], "task": [4, 9, 10], "gener": [4, 10], "speak": 4, "But": 4, "attr": 4, "maintain_propag": 4, "propag": 4, "sentinel": 4, "effect": [4, 11], "overrid": [4, 8, 10], "default": [4, 18], "behavior": 4, "whenev": 4, "call": [4, 10], "At": [4, 17], "time": [4, 9, 17], "write": [4, 9, 15], "onli": [4, 9, 10, 18], "unit": 4, "test": [4, 18], "instanc": [4, 9, 10], "identifi": 4, "add": [4, 9, 10], "suffix": 4, "For": [4, 10, 11, 12, 13, 17], "when": [4, 8, 9, 18], "relai": 4, "messag": [4, 9, 10], "other": 4, "than": 4, "trigger": 4, "run": [4, 9, 13], "close": 4, "hdfssensor": [6, 9], "basesensoroper": [6, 8], "webhdfssensor": [6, 8, 9], "hdfsregexsensor": [6, 9], "hdfsfoldersensor": 6, "filepath": [8, 17], "wait": 8, "land": 8, "template_field": [8, 10], "sequenc": [8, 10], "poke": [8, 17], "deriv": 8, "apach": [9, 13], "airflow": [9, 13, 14, 18], "provid": [9, 13, 14, 17, 18], "hdf": [9, 13, 15], "avail": [9, 13], "7": [9, 10, 12], "explain": 9, "support": [9, 10, 12], "polici": [9, 18], "bump": [9, 10], "minimum": [9, 10, 12], "39240": [9, 10], "turn": [9, 10], "pydant": [9, 10], "option": [9, 10, 11], "37320": [9, 10], "_read": [9, 10], "method": [9, 10], "after": [9, 10], "min": [9, 10], "6": [9, 10], "36425": [9, 10], "consolid": [9, 10], "load": [9, 10], "delete_local_log": [9, 10], "conf": [9, 10], "36422": [9, 10], "36145": [9, 10], "36017": [9, 10], "5": 9, "34728": [9, 10], "34511": [9, 10], "packag": 9, "name": [9, 10], "33813": [9, 10], "drop": [9, 10, 13], "python": [9, 10, 11, 12, 13], "abil": 9, "31512": [9, 10], "origin": [9, 13], "It": [9, 13], "old": [9, 10], "snakebit": [9, 10], "py3": [9, 10, 13], "had": 9, "updat": [9, 10], "year": 9, "protobuf": [9, 10], "thei": 9, "reach": 9, "end": 9, "life": 9, "still": 9, "need": 9, "have": [9, 10, 14, 18], "anymor": 9, "them": 9, "recommend": [9, 13], "switch": [9, 10], "new": [9, 10, 18], "api": [9, 17], "31262": [9, 10], "requir": [9, 18], "its": [9, 10], "june": [9, 10], "2023": [9, 10], "stop": 9, "would": 9, "like": 9, "due": 9, "incompat": [9, 10], "do": [9, 13], "one": [9, 13], "two": 9, "thing": 9, "protocol_buffers_python_implement": 9, "20": [9, 10], "make": [9, 10], "much": 9, "slower": 9, "includ": [9, 10, 12, 18], "multipl": [9, 11], "googl": 9, "kubernet": 9, "Of": 9, "some": [9, 14], "exampl": [9, 13, 14], "grpc": 9, "those": [9, 13], "well": [9, 13], "both": 9, "should": [9, 11, 13, 18], "treat": 9, "temporari": [9, 13], "workaround": 9, "migrat": 9, "soon": 9, "possibl": [9, 10, 11], "30917": [9, 10], "haclient": [9, 10], "invalid": [9, 10], "30164": [9, 10], "move": [9, 10], "27196": [9, 10], "ad": [9, 10], "25110": [9, 10], "bugfix": [9, 10, 18], "port": [9, 10, 11], "24550": [9, 10], "http": [9, 10, 11, 13], "github": 9, "com": [9, 13], "blob": 9, "main": [9, 13, 18], "readm": [9, 10], "md": 9, "chore": [9, 10], "refactor": [9, 10], "clean": [9, 10], "24219": [9, 10], "mistakenli": 9, "install_requir": 9, "22382": 9, "trove": 9, "classifi": [9, 10], "pypi": [9, 10, 18], "framework": 9, "how": [9, 10, 11, 13], "guid": [9, 10, 13], "oper": [9, 10, 11], "21393": [9, 10], "restor": [9, 10], "19711": [9, 10], "get_connect": [9, 10], "deprec": [9, 10], "warn": [9, 10, 13], "18331": [9, 10], "allow": [9, 10], "ssl": [9, 10, 11], "17637": [9, 10], "optimis": 9, "import": [9, 10, 13], "auto": [9, 10], "appli": [9, 10], "apply_default": [9, 10], "decor": [9, 10], "15667": [9, 10], "first": [9, 10], "upgrad": [9, 10, 18], "least": 9, "otherwis": 9, "automat": 9, "manual": 9, "db": 9, "complet": 9, "document": [9, 10], "initi": [9, 10], "hadoop": [10, 12, 17], "distribut": [10, 12, 13], "system": [10, 12], "detail": [10, 13], "commit": 10, "list": [10, 11], "chang": [10, 18], "high": 10, "level": 10, "changelog": 10, "see": [10, 12], "inform": [10, 11, 17, 18], "2024": 10, "04": 10, "25": 10, "subject": 10, "ead9b00f7c": 10, "5fa80b6aea": 10, "prepar": [10, 18], "doc": [10, 11], "1st": 10, "wave": 10, "rc1": 10, "april": 10, "38863": 10, "e706416cd5": 10, "06": 10, "fix": [10, 18], "try002": 10, "38792": 10, "0a74928894": 10, "03": 10, "18": 10, "ruff": 10, "38240": 10, "83316b8158": 10, "march": 10, "37876": 10, "5a0be392e6": 10, "02": 10, "16": 10, "comment": 10, "about": [10, 13, 18], "manag": [10, 13], "37488": 10, "c3f48eeda0": 10, "13": 10, "bfb054e9e8": 10, "12": [10, 13], "februari": 10, "37326": 10, "cead3da4a6": 10, "01": 10, "26": 10, "rc2": 10, "2nd": 10, "round": 10, "jan": 10, "37019": 10, "0b680c9492": 10, "revert": 10, "logger_nam": 10, "param": 10, "order": 10, "logger": 10, "36675": 10, "37015": 10, "2b4da0101f": 10, "22": 10, "januari": 10, "36945": 10, "6bd450da1e": 10, "19ebcac239": 10, "07": 10, "36640": 10, "6937ae7647": 10, "30": 10, "speed": 10, "up": 10, "autocomplet": 10, "breez": 10, "simplifi": 10, "state": 10, "36499": 10, "28": 10, "9b5d6bfe27": 10, "3rd": 10, "deember": 10, "36464": 10, "dd96be5468": 10, "d986bed292": 10, "23": 10, "b15d5578da": 10, "re": 10, "decemb": 10, "36380": 10, "f5883d6e7b": 10, "36373": 10, "6ee4d402e5": 10, "08": 10, "999b70178a": 10, "36112": 10, "d0918d77ee": 10, "0b23d5601c": 10, "11": [10, 13], "24": 10, "novemb": 10, "35836": 10, "99534e47f3": 10, "19": 10, "reproduc": 10, "build": [10, 13], "35693": 10, "99df205f42": 10, "reappli": 10, "templat": 10, "35686": 10, "1b059c57d6": 10, "35537": 10, "3b8db612ff": 10, "qubol": 10, "35492": 10, "d1c58d86de": 10, "octob": 10, "35233": 10, "3592ff4046": 10, "35187": 10, "dd7ba3cae1": 10, "pre": 10, "292": 10, "35053": 10, "e9987d5059": 10, "34916": 10, "0c8e30e43b": 10, "05": 10, "659d94f0ae": 10, "09": 10, "21": 10, "21990ed894": 10, "34201": 10, "d96f931b99": 10, "correct": [10, 13], "head": 10, "33809": 10, "dfa70b9423": 10, "c077d19060": 10, "aug": 10, "33730": 10, "a54c2424df": 10, "typo": 10, "doubl": 10, "word": 10, "33623": 10, "c645d8e40c": 10, "d401": 10, "airbyt": 10, "atlassian": 10, "inclus": 10, "33354": 10, "b5a4d36383": 10, "33291": 10, "225e3041d2": 10, "juli": 10, "32381": 10, "3878fe6fab": 10, "spuriou": 10, "header": 10, "32373": 10, "cb4927a018": 10, "32298": 10, "8c37b74a20": 10, "d205": 10, "common": 10, "32226": 10, "09d4718d3a": 10, "27": 10, "improv": 10, "structur": 10, "32125": 10, "79bcc2e668": 10, "32001": 10, "8b146152d6": 10, "32015": 10, "d3c8881cc6": 10, "store": [10, 14, 17], "0261bc57bc": 10, "31756": 10, "a473facf6c": 10, "d400": 10, "pydocstyl": 10, "31424": 10, "45548b9451": 10, "mai": 10, "31416": 10, "abea189022": 10, "31393": 10, "f5aed58d9f": 10, "circular": 10, "error": 10, "caus": 10, "31379": 10, "d9ff55cf6d": 10, "31252": 10, "9a8c2044bd": 10, "15": 10, "eef5bc7f16": 10, "full": 10, "autom": 10, "30994": 10, "a7eb32a5b2": 10, "d23a3bbed8": 10, "mechan": 10, "suspend": 10, "30422": 10, "55dbf1ff1f": 10, "30378": 10, "e141699c8a": 10, "2022": 10, "12c3c39d1a": 10, "27613": 10, "9ab1a6a3e7": 10, "style": 10, "26872": 10, "78b8ea2f22": 10, "2a34dc9e84": 10, "enabl": [10, 11], "normal": 10, "format": 10, "27205": 10, "f8db64c35c": 10, "septemb": 10, "26731": 10, "06acf40a43": 10, "pep": 10, "563": 10, "postpon": 10, "evalu": 10, "annot": 10, "non": [10, 11], "core": 10, "26289": 10, "e5ac6c7cfb": 10, "august": 10, "25618": 10, "52cf579352": 10, "d2459a241b": 10, "25030": 10, "0de31bd73a": 10, "29": 10, "24672": 10, "510a6bab45": 10, "class": [10, 12], "yaml": 10, "24702": 10, "d8ec1ec8ec": 10, "dcdcf3a2b8": 10, "24307": 10, "717a7588bc": 10, "descript": 10, "24292": 10, "aeabe994b3": 10, "24231": 10, "b4a5783a2a": 10, "027b707d21": 10, "explanatori": 10, "contributor": 10, "24229": 10, "d7dbfb7e26": 10, "22383": 10, "14": 10, "16adc035b1": 10, "22226": 10, "f5b96315fe": 10, "feb": 10, "22056": 10, "837ff7ed10": 10, "6c3a67d4fc": 10, "2021": [10, 13], "21257": 10, "602abe8394": 10, "line": 10, "now": 10, "sphinx": 10, "autoapi": 10, "typehint": 10, "20951": 10, "f77417eb0d": 10, "31": 10, "k8": 10, "compat": 10, "20614": 10, "97496ba2b4": 10, "20523": 10, "d56e7b56bb": 10, "mypi": 10, "friendli": 10, "20571": 10, "485ff6cc64": 10, "20422": 10, "853576d901": 10, "19882": 10, "1b97d654ae": 10, "840ea3efb9": 10, "18613": 10, "2b62a75a34": 10, "0a68588479": 10, "17890": 10, "be75dcd39c": 10, "meta": 10, "data": [10, 17], "76ed2a49c6": 10, "lazili": 10, "individu": 10, "17682": 10, "0016007b86": 10, "87f408b1e7": 10, "17116": 10, "b916b75079": 10, "17015": 10, "866a601b76": 10, "pylint": 10, "our": 10, "toolchain": 10, "16682": 10, "bbc627a3da": 10, "16501": 10, "cbf8001d76": 10, "synchron": 10, "buggfix": 10, "16464": 10, "1fba5402bb": 10, "more": [10, 11], "16405": 10, "9c94b72d44": 10, "16294": 10, "37681bca00": 10, "807ad32ce5": 10, "15576": 10, "71c673e427": 10, "docstr": 10, "modul": 10, "miss": 10, "15391": 10, "6f8ab9e6da": 10, "python2": 10, "relat": 10, "handl": 10, "15301": 10, "68e4c4dcb0": 10, "backport": 10, "14886": 10, "6e6526a0f6": 10, "broken": 10, "14734": 10, "71c5979a17": 10, "11212": 10, "88bdcfa0df": 10, "14013": 10, "ac2f72c98d": 10, "implement": 10, "tool": 10, "13767": 10, "a9ac2b040b": 10, "f": 10, "flynt": 10, "13732": 10, "295d66f914": 10, "2020": 10, "grammar": 10, "13380": 10, "6cf76d7ac0": 10, "command": 10, "13148": 10, "5090fb0c89": 10, "script": [10, 13], "integr": 10, "json": [10, 11], "13073": 10, "32971a1a2d": 10, "12955": 10, "b40dffa085": 10, "renam": 10, "rema": 10, "match": [10, 13], "aip": 10, "12917": 10, "9b39f24780": 10, "dynam": 10, "form": 10, "field": 10, "per": 10, "12558": 10, "370e7d07d1": 10, "12513": 10, "c34ef853c8": 10, "separ": [10, 11, 18], "out": [10, 18], "12444": 10, "0080354502": 10, "0b2": 10, "batch": 10, "12449": 10, "ae7cb4a1e2": 10, "17": 10, "wrong": 10, "hash": 10, "12390": 10, "6889a333cf": 10, "ref": 10, "12366": 10, "7825e8f590": 10, "12304": 10, "85a18e13d9": 10, "point": 10, "project": 10, "page": [10, 13], "cross": 10, "12212": 10, "59eb5de78c": 10, "come": 10, "0beta1": 10, "12206": 10, "b2a28d1590": 10, "dev": 10, "12082": 10, "41bf172c1d": 10, "express": 10, "12093": 10, "4e8f9cc8d0": 10, "black": 10, "formmatt": 10, "9550": 10, "8c42cf1b00": 10, "pyupgrad": 10, "11447": 10, "5a439e84eb": 10, "2a1": 10, "11855": 10, "872b1566a1": 10, "setup": [10, 14], "11826": 10, "349b0811c3": 10, "d200": 10, "11688": 10, "16e7129719": 10, "11487": 10, "0a0e1af800": 10, "markdown": 10, "link": [10, 13], "toc": 10, "11249": 10, "ca4238eb4d": 10, "month": 10, "11242": 10, "5220e4c384": 10, "11238": 10, "fdd9b6f65b": 10, "10543": 10, "d1bce91bb2": 10, "d403": 10, "capit": 10, "10530": 10, "3696c34c28": 10, "10528": 10, "2f2d8dbfaf": 10, "noinspect": 10, "nativ": [10, 17], "intellij": 10, "10525": 10, "ee7ca128a1": 10, "refernc": 10, "10483": 10, "3b3287d7ac": 10, "enforc": 10, "10170": 10, "33f0cd2657": 10, "keep": 10, "signatur": [10, 13], "9784": 10, "4d74ac2111": 10, "increas": 10, "9729": 10, "44d4ae809c": 10, "9686": 10, "d0e7db4024": 10, "fresh": 10, "9408": 10, "12af6a0800": 10, "final": 10, "cleanup": 10, "23rc1": 10, "9404": 10, "c7e5bce57f": 10, "candid": 10, "9370": 10, "f6bd817a3a": 10, "introduc": 10, "transfer": 10, "9320": 10, "0b0e4f7a4c": 10, "rc3": 10, "relas": 10, "9026": 10, "00642a46d0": 10, "remain": 10, "wrongli": 10, "8994": 10, "375d1ca229": 10, "8898": 10, "12c5e5d8a": 10, "8891": 10, "f3521fb0e3": 10, "regener": 10, "8886": 10, "92585ca4cb": 10, "8807": 10, "87969a350d": 10, "6515": 10, "info": 10, "8170": 10, "6c670870aa": 10, "6833": 10, "7454": 10, "7e6372a681": 10, "super": 10, "7820": 10, "f3ad5cf618": 10, "4681": 10, "7309": 10, "97a429f9d0": 10, "6714": 10, "magic": 10, "utf": 10, "8": 10, "7338": 10, "cf141506a2": 10, "6708": 10, "uniqu": 10, "7330": 10, "0481b9a957": 10, "6539": 10, "7142": 10, "host": 11, "yarn": 11, "url": [11, 14], "specifi": 11, "comma": 11, "case": [11, 18], "login": 11, "kerber": 11, "extra": 11, "dictionari": 11, "follow": [11, 13, 14, 18], "use_ssl": 11, "By": [11, 13], "verifi": 11, "refer": 11, "request": 11, "org": [11, 13], "en": 11, "master": 11, "advanc": 11, "cert": 11, "verif": 11, "top": [12, 13], "avro": 12, "datafram": 12, "kerbero": 12, "describ": 13, "download": 13, "offici": 13, "also": 13, "most": 13, "choos": 13, "differ": 13, "select": 13, "down": 13, "left": 13, "sdist": 13, "whl": 13, "checksum": 13, "softwar": 13, "foundat": 13, "asc": 13, "sha512": 13, "code": 13, "abov": [13, 14], "contain": 13, "pgp": 13, "kei": 13, "essenti": 13, "sha": 13, "gpg": 13, "relev": 13, "get": [13, 18], "mirror": 13, "pgpk": 13, "ka": 13, "To": [13, 14], "binari": 13, "pgpv": 13, "tar": 13, "gz": 13, "made": 13, "sat": 13, "sep": 13, "49": 13, "54": 13, "bst": 13, "rsa": 13, "cde15c6e4d3a8ec4ecf4ba4b6674e08ad7de406f": 13, "issuer": 13, "kaxilnaik": 13, "good": [13, 18], "kaxil": 13, "naik": 13, "unknown": 13, "aka": 13, "gmail": 13, "certifi": 13, "trust": 13, "There": 13, "indic": 13, "belong": 13, "owner": 13, "primari": 13, "fingerprint": 13, "cde1": 13, "5c6e": 13, "4d3a": 13, "8ec4": 13, "ecf4": 13, "ba4b": 13, "6674": 13, "e08a": 13, "d7de": 13, "406f": 13, "worri": 13, "certif": 13, "self": 13, "sign": 13, "why": 13, "server": 13, "previou": 13, "step": 13, "know": 13, "valid": 13, "sum": 13, "shasum": 13, "512": 13, "diff": 13, "bin": 13, "bash": 13, "package_vers": 13, "package_nam": 13, "provider_download_dir": 13, "mktemp": 13, "d": 13, "dep": 13, "dest": 13, "curl": 13, "apache_airflow_providers_apache_hdf": 13, "l": 13, "o": 13, "echo": 13, "la": 13, "onc": 13, "instruct": [13, 18], "chapter": 13, "don": 14, "t": 14, "properli": 14, "process": 14, "fail": 14, "cfg": 14, "must": 14, "configur": 14, "suppli": 14, "locat": 14, "start": 14, "either": 14, "access": [14, 17], "storag": 14, "remote_log": 14, "remote_base_log_fold": 14, "remote_log_conn_id": 14, "In": 14, "try": 14, "servic": 17, "same": 17, "retain": 17, "protocol": 17, "offer": 17, "better": 17, "throughput": 17, "until": 17, "found": [17, 18], "further": 17, "look": 17, "rest": 17, "independ": 18, "itself": 18, "vulner": 18, "publish": 18, "we": 18, "develop": 18, "alwai": 18, "done": 18, "branch": 18, "where": 18, "next": 18, "strict": 18, "semver": 18, "scope": 18, "major": 18, "break": 18, "minor": 18, "patchlevel": 18, "bug": 18, "receiv": 18, "so": 18, "rule": 18, "critic": 18, "reason": 18, "band": 18, "which": 18, "stakehold": 18, "might": 18, "decid": 18, "cherri": 18, "pick": 18, "older": 18, "mix": 18, "govern": 18, "model": 18, "interest": 18, "parti": 18}, "objects": {"airflow.providers.apache": [[3, 0, 0, "-", "hdfs"]], "airflow.providers.apache.hdfs": [[3, 1, 1, "", "__version__"], [1, 0, 0, "-", "hooks"], [5, 0, 0, "-", "log"], [7, 0, 0, "-", "sensors"]], "airflow.providers.apache.hdfs.hooks": [[0, 0, 0, "-", "hdfs"], [2, 0, 0, "-", "webhdfs"]], "airflow.providers.apache.hdfs.hooks.hdfs": [[0, 2, 1, "", "HDFSHook"], [0, 3, 1, "", "HDFSHookException"]], "airflow.providers.apache.hdfs.hooks.webhdfs": [[2, 3, 1, "", "AirflowWebHDFSHookException"], [2, 2, 1, "", "WebHDFSHook"], [2, 1, 1, "", "log"]], "airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook": [[2, 4, 1, "", "check_for_path"], [2, 5, 1, "", "conn_name_attr"], [2, 5, 1, "", "conn_type"], [2, 5, 1, "", "default_conn_name"], [2, 4, 1, "", "get_conn"], [2, 5, 1, "", "hook_name"], [2, 4, 1, "", "load_file"], [2, 4, 1, "", "read_file"]], "airflow.providers.apache.hdfs.log": [[4, 0, 0, "-", "hdfs_task_handler"]], "airflow.providers.apache.hdfs.log.hdfs_task_handler": [[4, 2, 1, "", "HdfsTaskHandler"]], "airflow.providers.apache.hdfs.log.hdfs_task_handler.HdfsTaskHandler": [[4, 4, 1, "", "close"], [4, 4, 1, "", "hook"], [4, 4, 1, "", "set_context"]], "airflow.providers.apache.hdfs.sensors": [[6, 0, 0, "-", "hdfs"], [8, 0, 0, "-", "web_hdfs"]], "airflow.providers.apache.hdfs.sensors.hdfs": [[6, 2, 1, "", "HdfsFolderSensor"], [6, 2, 1, "", "HdfsRegexSensor"], [6, 2, 1, "", "HdfsSensor"]], "airflow.providers.apache.hdfs.sensors.web_hdfs": [[8, 2, 1, "", "WebHdfsSensor"]], "airflow.providers.apache.hdfs.sensors.web_hdfs.WebHdfsSensor": [[8, 4, 1, "", "poke"], [8, 5, 1, "", "template_fields"]]}, "objtypes": {"0": "py:module", "1": "py:data", "2": "py:class", "3": "py:exception", "4": "py:method", "5": "py:attribute"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "data", "Python data"], "2": ["py", "class", "Python class"], "3": ["py", "exception", "Python exception"], "4": ["py", "method", "Python method"], "5": ["py", "attribute", "Python attribute"]}, "titleterms": {"airflow": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12], "provid": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12], "apach": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 16], "hdf": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 14, 16, 17], "hook": [0, 1, 2], "modul": [0, 2, 4, 6, 8], "content": [0, 2, 3, 4, 6, 8], "class": [0, 2, 4, 6, 8], "submodul": [1, 5, 7], "webhdf": [2, 17], "attribut": 2, "subpackag": 3, "packag": [3, 10, 12, 13], "log": [4, 5, 14, 15], "hdfs_task_handl": 4, "sensor": [6, 7, 8], "web_hdf": 8, "changelog": 9, "4": [9, 10], "0": [9, 10], "misc": 9, "3": [9, 10], "2": [9, 10], "1": [9, 10], "bug": 9, "fix": 9, "featur": 9, "break": 9, "chang": 9, "connect": 11, "default": 11, "id": 11, "configur": 11, "instal": [12, 13], "requir": 12, "from": 13, "sourc": 13, "releas": [13, 18], "integr": 13, "verifi": 13, "pypi": 13, "write": 14, "enabl": 14, "remot": 14, "task": 15, "hadoop": 16, "oper": [16, 17], "webhdfssensor": 17, "wait": 17, "file": 17, "folder": 17, "land": 17, "refer": 17, "secur": 18, "patch": 18}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx.ext.intersphinx": 1, "sphinx": 57}, "alltitles": {"airflow.providers.apache.hdfs.hooks.hdfs": [[0, "module-airflow.providers.apache.hdfs.hooks.hdfs"]], "Module Contents": [[0, "module-contents"], [2, "module-contents"], [4, "module-contents"], [6, "module-contents"], [8, "module-contents"]], "Classes": [[0, "classes"], [2, "classes"], [4, "classes"], [6, "classes"], [8, "classes"]], "airflow.providers.apache.hdfs.hooks": [[1, "module-airflow.providers.apache.hdfs.hooks"]], "Submodules": [[1, "submodules"], [5, "submodules"], [7, "submodules"]], "airflow.providers.apache.hdfs.hooks.webhdfs": [[2, "module-airflow.providers.apache.hdfs.hooks.webhdfs"]], "Attributes": [[2, "attributes"]], "airflow.providers.apache.hdfs": [[3, "module-airflow.providers.apache.hdfs"]], "Subpackages": [[3, "subpackages"]], "Package Contents": [[3, "package-contents"]], "airflow.providers.apache.hdfs.log.hdfs_task_handler": [[4, "module-airflow.providers.apache.hdfs.log.hdfs_task_handler"]], "airflow.providers.apache.hdfs.log": [[5, "module-airflow.providers.apache.hdfs.log"]], "airflow.providers.apache.hdfs.sensors.hdfs": [[6, "module-airflow.providers.apache.hdfs.sensors.hdfs"]], "airflow.providers.apache.hdfs.sensors": [[7, "module-airflow.providers.apache.hdfs.sensors"]], "airflow.providers.apache.hdfs.sensors.web_hdfs": [[8, "module-airflow.providers.apache.hdfs.sensors.web_hdfs"]], "Changelog": [[9, "changelog"]], "4.4.0": [[9, "id1"], [10, "id1"]], "Misc": [[9, "misc"], [9, "id3"], [9, "id5"], [9, "id9"], [9, "id12"], [9, "id14"], [9, "id18"], [9, "id23"], [9, "id30"], [9, "id34"], [9, "id36"], [9, "id43"]], "4.3.3": [[9, "id2"], [10, "id2"]], "4.3.2": [[9, "id4"], [10, "id3"]], "4.3.1": [[9, "id6"], [10, "id4"]], "Bug Fixes": [[9, "bug-fixes"], [9, "id20"], [9, "id27"], [9, "id32"], [9, "id40"]], "4.3.0": [[9, "id7"], [10, "id5"]], "4.2.0": [[9, "id10"], [10, "id6"]], "4.1.1": [[9, "id13"], [10, "id7"]], "4.1.0": [[9, "id15"], [10, "id8"]], "Features": [[9, "features"], [9, "id25"], [9, "id38"], [9, "id42"]], "4.0.0": [[9, "id16"], [10, "id9"]], "Breaking changes": [[9, "breaking-changes"], [9, "id29"], [9, "id45"]], "3.2.1": [[9, "id19"], [10, "id10"]], "3.2.0": [[9, "id21"], [10, "id11"]], "3.1.0": [[9, "id24"], [10, "id12"]], "3.0.1": [[9, "id26"], [10, "id13"]], "3.0.0": [[9, "id28"], [10, "id14"]], "2.2.3": [[9, "id31"], [10, "id15"]], "2.2.2": [[9, "id33"], [10, "id16"]], "2.2.1": [[9, "id35"], [10, "id17"]], "2.2.0": [[9, "id37"], [10, "id18"]], "2.1.1": [[9, "id39"], [10, "id19"]], "2.1.0": [[9, "id41"], [10, "id20"]], "2.0.0": [[9, "id44"], [10, "id21"]], "1.0.1": [[9, "id46"], [10, "id22"]], "1.0.0": [[9, "id47"], [10, "id23"]], "Package apache-airflow-providers-apache-hdfs": [[10, "package-apache-airflow-providers-apache-hdfs"]], "Apache HDFS Connection": [[11, "apache-hdfs-connection"]], "Default Connection IDs": [[11, "default-connection-ids"]], "Configuring the Connection": [[11, "configuring-the-connection"]], "apache-airflow-providers-apache-hdfs": [[12, "apache-airflow-providers-apache-hdfs"]], "apache-airflow-providers-apache-hdfs package": [[12, "apache-airflow-providers-apache-hdfs-package"]], "Provider package": [[12, "provider-package"]], "Installation": [[12, "installation"]], "Requirements": [[12, "requirements"]], "Installing from sources": [[13, "installing-from-sources"]], "Released packages": [[13, "released-packages"]], "Release integrity": [[13, "release-integrity"]], "Verifying PyPI releases": [[13, "verifying-pypi-releases"]], "Writing logs to HDFS": [[14, "writing-logs-to-hdfs"]], "Enabling remote logging": [[14, "enabling-remote-logging"]], "Logging for Tasks": [[15, "logging-for-tasks"]], "Apache Hadoop HDFS Operators": [[16, "apache-hadoop-hdfs-operators"]], "WebHDFS Operators": [[17, "webhdfs-operators"]], "WebHdfsSensor": [[17, "webhdfssensor"]], "Waits for a file or folder to land in HDFS": [[17, "waits-for-a-file-or-folder-to-land-in-hdfs"]], "Reference": [[17, "reference"]], "Releasing security patches": [[18, "releasing-security-patches"]]}, "indexentries": {"hdfshook (class in airflow.providers.apache.hdfs.hooks.hdfs)": [[0, "airflow.providers.apache.hdfs.hooks.hdfs.HDFSHook"]], "hdfshookexception": [[0, "airflow.providers.apache.hdfs.hooks.hdfs.HDFSHookException"]], "airflow.providers.apache.hdfs.hooks.hdfs": [[0, "module-airflow.providers.apache.hdfs.hooks.hdfs"]], "module": [[0, "module-airflow.providers.apache.hdfs.hooks.hdfs"], [1, "module-airflow.providers.apache.hdfs.hooks"], [2, "module-airflow.providers.apache.hdfs.hooks.webhdfs"], [3, "module-airflow.providers.apache.hdfs"], [4, "module-airflow.providers.apache.hdfs.log.hdfs_task_handler"], [5, "module-airflow.providers.apache.hdfs.log"], [6, "module-airflow.providers.apache.hdfs.sensors.hdfs"], [7, "module-airflow.providers.apache.hdfs.sensors"], [8, "module-airflow.providers.apache.hdfs.sensors.web_hdfs"]], "airflow.providers.apache.hdfs.hooks": [[1, "module-airflow.providers.apache.hdfs.hooks"]], "airflowwebhdfshookexception": [[2, "airflow.providers.apache.hdfs.hooks.webhdfs.AirflowWebHDFSHookException"]], "webhdfshook (class in airflow.providers.apache.hdfs.hooks.webhdfs)": [[2, "airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook"]], "airflow.providers.apache.hdfs.hooks.webhdfs": [[2, "module-airflow.providers.apache.hdfs.hooks.webhdfs"]], "check_for_path() (airflow.providers.apache.hdfs.hooks.webhdfs.webhdfshook method)": [[2, "airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook.check_for_path"]], "conn_name_attr (airflow.providers.apache.hdfs.hooks.webhdfs.webhdfshook attribute)": [[2, "airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook.conn_name_attr"]], "conn_type (airflow.providers.apache.hdfs.hooks.webhdfs.webhdfshook attribute)": [[2, "airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook.conn_type"]], "default_conn_name (airflow.providers.apache.hdfs.hooks.webhdfs.webhdfshook attribute)": [[2, "airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook.default_conn_name"]], "get_conn() (airflow.providers.apache.hdfs.hooks.webhdfs.webhdfshook method)": [[2, "airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook.get_conn"]], "hook_name (airflow.providers.apache.hdfs.hooks.webhdfs.webhdfshook attribute)": [[2, "airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook.hook_name"]], "load_file() (airflow.providers.apache.hdfs.hooks.webhdfs.webhdfshook method)": [[2, "airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook.load_file"]], "log (in module airflow.providers.apache.hdfs.hooks.webhdfs)": [[2, "airflow.providers.apache.hdfs.hooks.webhdfs.log"]], "read_file() (airflow.providers.apache.hdfs.hooks.webhdfs.webhdfshook method)": [[2, "airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook.read_file"]], "__version__ (in module airflow.providers.apache.hdfs)": [[3, "airflow.providers.apache.hdfs.__version__"]], "airflow.providers.apache.hdfs": [[3, "module-airflow.providers.apache.hdfs"]], "hdfstaskhandler (class in airflow.providers.apache.hdfs.log.hdfs_task_handler)": [[4, "airflow.providers.apache.hdfs.log.hdfs_task_handler.HdfsTaskHandler"]], "airflow.providers.apache.hdfs.log.hdfs_task_handler": [[4, "module-airflow.providers.apache.hdfs.log.hdfs_task_handler"]], "close() (airflow.providers.apache.hdfs.log.hdfs_task_handler.hdfstaskhandler method)": [[4, "airflow.providers.apache.hdfs.log.hdfs_task_handler.HdfsTaskHandler.close"]], "hook() (airflow.providers.apache.hdfs.log.hdfs_task_handler.hdfstaskhandler method)": [[4, "airflow.providers.apache.hdfs.log.hdfs_task_handler.HdfsTaskHandler.hook"]], "set_context() (airflow.providers.apache.hdfs.log.hdfs_task_handler.hdfstaskhandler method)": [[4, "airflow.providers.apache.hdfs.log.hdfs_task_handler.HdfsTaskHandler.set_context"]], "airflow.providers.apache.hdfs.log": [[5, "module-airflow.providers.apache.hdfs.log"]], "hdfsfoldersensor (class in airflow.providers.apache.hdfs.sensors.hdfs)": [[6, "airflow.providers.apache.hdfs.sensors.hdfs.HdfsFolderSensor"]], "hdfsregexsensor (class in airflow.providers.apache.hdfs.sensors.hdfs)": [[6, "airflow.providers.apache.hdfs.sensors.hdfs.HdfsRegexSensor"]], "hdfssensor (class in airflow.providers.apache.hdfs.sensors.hdfs)": [[6, "airflow.providers.apache.hdfs.sensors.hdfs.HdfsSensor"]], "airflow.providers.apache.hdfs.sensors.hdfs": [[6, "module-airflow.providers.apache.hdfs.sensors.hdfs"]], "airflow.providers.apache.hdfs.sensors": [[7, "module-airflow.providers.apache.hdfs.sensors"]], "webhdfssensor (class in airflow.providers.apache.hdfs.sensors.web_hdfs)": [[8, "airflow.providers.apache.hdfs.sensors.web_hdfs.WebHdfsSensor"]], "airflow.providers.apache.hdfs.sensors.web_hdfs": [[8, "module-airflow.providers.apache.hdfs.sensors.web_hdfs"]], "poke() (airflow.providers.apache.hdfs.sensors.web_hdfs.webhdfssensor method)": [[8, "airflow.providers.apache.hdfs.sensors.web_hdfs.WebHdfsSensor.poke"]], "template_fields (airflow.providers.apache.hdfs.sensors.web_hdfs.webhdfssensor attribute)": [[8, "airflow.providers.apache.hdfs.sensors.web_hdfs.WebHdfsSensor.template_fields"]]}}) |