fix wasb parsing blob as string instead of bytes

.pre-commit-config.yaml

@@ -563,18 +563,6 @@ repos:
         language_version: python3
         additional_dependencies: ['flynt']
         files: \.py$
-      - id: ui-lint
-        name: Lint React UI
-        language: node
-        entry: bash
-        args: ["-c", "cd airflow/ui && yarn install && yarn lint"]
-        files: ^airflow/ui/.*\.(ts|tsx)$
-      - id: ui-test
-        name: Test React UI
-        language: node
-        entry: bash
-        args: ["-c", "cd airflow/ui && yarn install && yarn test"]
-        files: ^airflow/ui/.*\.(ts|tsx)$
         ## ADD MOST PRE-COMMITS ABOVE THAT LINE
         # The below pre-commits are those requiring CI image to be built
       - id: build

BREEZE.rst

@@ -2298,8 +2298,8 @@ This is the current syntax for  `./breeze <./breeze>`_:
                  pre-commit-hook-names provide-create-sessions providers-init-file provider-yamls
                  pydevd pydocstyle pylint pylint-tests python-no-log-warn pyupgrade
                  restrict-start_date rst-backticks setup-order setup-extra-packages shellcheck
-                 sort-in-the-wild sort-spelling-wordlist stylelint trailing-whitespace ui-lint
-                 ui-test update-breeze-file update-extras update-local-yml-file update-setup-cfg-file
+                 sort-in-the-wild sort-spelling-wordlist stylelint trailing-whitespace
+                 update-breeze-file update-extras update-local-yml-file update-setup-cfg-file
                  version-sync yamllint
 
         You can pass extra arguments including options to the pre-commit framework as

STATIC_CODE_CHECKS.rst

@@ -170,10 +170,6 @@ require Breeze Docker images to be installed locally:
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``trailing-whitespace``               Removes trailing whitespace at end of line
 ----------------------------------- ---------------------------------------------------------------- ------------
-``ui-lint``                           Lint the React UI
------------------------------------ ---------------------------------------------------------------- ------------
-``ui-test``                           Test the React UI
------------------------------------ ---------------------------------------------------------------- ------------
 ``update-breeze-file``                Update output of breeze command in BREEZE.rst
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``update-extras``                     Updates extras in the documentation

airflow/providers/microsoft/azure/log/wasb_task_handler.py

@@ -51,6 +51,7 @@ def __init__(
         self.remote_base = wasb_log_folder
         self.log_relative_path = ''
         self._hook = None
+        self.encoding = "UTF-8"
         self.closed = False
         self.upload_on_close = True
         self.delete_local_copy = delete_local_copy
@@ -159,7 +160,7 @@ def wasb_read(self, remote_log_location: str, return_error: bool = False):
         :type return_error: bool
         """
         try:
-            return self.hook.read_file(self.wasb_container, remote_log_location)
+            return self.hook.read_file(self.wasb_container, remote_log_location).decode(self.encoding)
         except AzureHttpError as e:
             msg = f'Could not read logs from {remote_log_location}'
             self.log.exception("Message: '%s', exception '%s'", msg, e)
@@ -181,11 +182,10 @@ def wasb_write(self, log: str, remote_log_location: str, append: bool = True) ->
             the new log is appended to any existing logs.
         :type append: bool
         """
-        if append and self.wasb_log_exists(remote_log_location):
-            old_log = self.wasb_read(remote_log_location)
-            log = '\n'.join([old_log, log]) if old_log else log
-
         try:
+            if append and self.wasb_log_exists(remote_log_location):
+                old_log = self.wasb_read(remote_log_location)
+                log = '\n'.join([old_log, log]) if old_log else log
             self.hook.load_string(
                 log,
                 self.wasb_container,

airflow/ui/.env.example

@@ -0,0 +1 @@
+API_URL = 'http://127.0.0.1:28080/api/v1/'

airflow/ui/.neutrinorc.js

@@ -20,6 +20,7 @@
 /*
   Config for running and building the app
 */
+require('dotenv').config();
 const typescript = require('neutrinojs-typescript');
 const typescriptLint = require('neutrinojs-typescript-eslint');
 const react = require('@neutrinojs/react');
@@ -37,6 +38,10 @@ module.exports = {
       // Aliases for internal modules
       neutrino.config.resolve.alias.set('root', resolve(__dirname));
       neutrino.config.resolve.alias.set('src', resolve(__dirname, 'src'));
+      neutrino.config.resolve.alias.set('views', resolve(__dirname, 'src/views'));
+      neutrino.config.resolve.alias.set('utils', resolve(__dirname, 'src/utils'));
+      neutrino.config.resolve.alias.set('auth', resolve(__dirname, 'src/auth'));
+      neutrino.config.resolve.alias.set('components', resolve(__dirname, 'src/components'));
     },
     typescript(),
     // Modify typescript config in .tsconfig.json
@@ -51,6 +56,9 @@ module.exports = {
       moduleDirectories: ['node_modules', 'src'],
     }),
     react({
+      env: [
+        'API_URL'
+      ],
       html: {
         title: 'Apache Airflow',
       }

airflow/ui/README.md

@@ -27,6 +27,25 @@
 - [Chakra UI](https://chakra-ui.com/) - a simple, modular and accessible component library that gives you all the building blocks you need to build your React applications.
 - [React Testing Library](https://testing-library.com/docs/react-testing-library/intro/) - write tests that focus on functionality instead of implementation
 
+## Environment variables
+
+To communicate with the API you need to adjust some environment variables for the webserver and this UI.
+
+Be sure to allow CORS headers and set up an auth backend on your Airflow instance.
+
+```
+export AIRFLOW__API__AUTH_BACKEND=airflow.api.auth.backend.basic_auth
+export AIRFLOW__API__ACCESS_CONTROL_ALLOW_HEADERS=*
+export AIRFLOW__API__ACCESS_CONTROL_ALLOW_METHODS=*
+export AIRFLOW__API__ACCESS_CONTROL_ALLOW_ORIGIN=http://127.0.0.1:28080/
+```
+
+Create your local environment and adjust the `API_URL` if needed.
+
+```bash
+cp .env.example .env
+```
+
 ## Installation
 
 Clone the repository and use the package manager [yarn](https://yarnpkg.com) to install dependencies and get the project running.

airflow/ui/package.json

@@ -13,10 +13,15 @@
     "@emotion/react": "^11.1.5",
     "@emotion/styled": "^11.1.5",
     "@neutrinojs/copy": "^9.5.0",
+    "axios": "^0.21.1",
+    "dotenv": "^8.2.0",
     "framer-motion": "^3.10.0",
     "react": "^16",
     "react-dom": "^16",
-    "react-hot-loader": "^4"
+    "react-hot-loader": "^4",
+    "react-icons": "^4.2.0",
+    "react-query": "^3.12.3",
+    "react-router-dom": "^5.2.0"
   },
   "devDependencies": {
     "@neutrinojs/eslint": "^9.5.0",
@@ -27,14 +32,18 @@
     "@types/jest": "^26.0.20",
     "@types/react": "^17.0.3",
     "@types/react-dom": "^17.0.2",
+    "@types/react-router-dom": "^5.1.7",
     "eslint": "^7",
     "eslint-config-airbnb-typescript": "^12.3.1",
     "eslint-plugin-import": "^2.22.1",
     "eslint-plugin-jsx-a11y": "^6.4.1",
+    "history": "^5.0.0",
     "jest": "^26",
     "neutrino": "^9.5.0",
     "neutrinojs-typescript": "^1.1.6",
     "neutrinojs-typescript-eslint": "^1.3.1",
+    "nock": "^13.0.11",
+    "react-test-renderer": "^17.0.1",
     "typescript": "^4.2.3",
     "webpack": "^4",
     "webpack-cli": "^3",

airflow/ui/src/App.tsx

@@ -19,12 +19,45 @@
 
 import { hot } from 'react-hot-loader';
 import React from 'react';
-import { Center, Heading } from '@chakra-ui/react';
+import { Route, Redirect, Switch } from 'react-router-dom';
+
+import PrivateRoute from 'auth/PrivateRoute';
+
+import Pipelines from 'views/Pipelines';
+import Pipeline from 'views/Pipeline';
+
+import EventLogs from 'views/Activity/EventLogs';
+
+import Config from 'views/Config';
+
+import Access from 'views/Access';
+import Users from 'views/Access/Users';
+import Roles from 'views/Access/Roles';
+
+import Docs from 'views/Docs';
+import NotFound from 'views/NotFound';
 
 const App = () => (
-  <Center height="100vh">
-    <Heading>Apache Airflow new UI</Heading>
-  </Center>
+  <Switch>
+    <Redirect exact path="/" to="/pipelines" />
+    <PrivateRoute exact path="/pipelines" component={Pipelines} />
+    <PrivateRoute exact path="/pipelines/:dagId" component={Pipeline} />
+
+    <PrivateRoute exact path="/activity/event-logs" component={EventLogs} />
+
+    <PrivateRoute exact path="/config" component={Config} />
+
+    <PrivateRoute exact path="/access" component={Access} />
+    <PrivateRoute exact path="/access/users" component={Users} />
+    <PrivateRoute exact path="/access/users/new" component={Users} />
+    <PrivateRoute exact path="/access/users/:username" component={Users} />
+    <PrivateRoute exact path="/access/users/:username/edit" component={Users} />
+    <PrivateRoute exact path="/access/roles" component={Roles} />
+
+    <Route exact path="/docs" component={Docs} />
+
+    <Route component={NotFound} />
+  </Switch>
 );
 
 export default hot(module)(App);

airflow/ui/src/auth/AuthProvider.tsx

@@ -0,0 +1,113 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React, {
+  useState, useEffect, useCallback, ReactNode, ReactElement,
+} from 'react';
+import axios from 'axios';
+import { useQueryClient } from 'react-query';
+
+import {
+  checkExpire, clearAuth, get, set,
+} from 'utils/localStorage';
+import { AuthContext } from './context';
+
+type Props = {
+  children: ReactNode;
+};
+
+const AuthProvider = ({ children }: Props): ReactElement => {
+  const [hasValidAuthToken, setHasValidAuthToken] = useState(false);
+  const [error, setError] = useState<Error | null>(null);
+  const [loading, setLoading] = useState(true);
+  const queryClient = useQueryClient();
+
+  const clearData = useCallback(() => {
+    setHasValidAuthToken(false);
+    clearAuth();
+    queryClient.clear();
+    axios.defaults.headers.common.Authorization = null;
+  }, [queryClient]);
+
+  const logout = () => clearData();
+
+  // intercept responses and logout on unauthorized error
+  axios.interceptors.response.use(
+    (res) => res,
+    (err) => {
+      if (err && err.response && err.response.status === 401) {
+        logout();
+      }
+      return Promise.reject(err);
+    },
+  );
+
+  useEffect(() => {
+    const token = get('token');
+    const isExpired = checkExpire('token');
+    if (token && !isExpired) {
+      axios.defaults.headers.common.Authorization = token;
+      setHasValidAuthToken(true);
+    } else if (token) {
+      clearData();
+      setError(new Error('Token invalid, please reauthenticate.'));
+    } else {
+      setHasValidAuthToken(false);
+    }
+    setLoading(false);
+  }, [clearData]);
+
+  // Login with basic auth.
+  // There is no actual auth endpoint yet, so we check against a generic endpoint
+  const login = async (username: string, password: string) => {
+    setLoading(true);
+    setError(null);
+    try {
+      const authorization = `Basic ${btoa(`${username}:${password}`)}`;
+      await axios.get(`${process.env.API_URL}config`, {
+        headers: {
+          Authorization: authorization,
+        },
+      });
+      set('token', authorization);
+      axios.defaults.headers.common.Authorization = authorization;
+      setLoading(false);
+      setHasValidAuthToken(true);
+    } catch (e) {
+      setLoading(false);
+      setError(e);
+    }
+  };
+
+  return (
+    <AuthContext.Provider
+      value={{
+        hasValidAuthToken,
+        logout,
+        login,
+        loading,
+        error,
+      }}
+    >
+      {children}
+    </AuthContext.Provider>
+  );
+};
+
+export default AuthProvider;

airflow/ui/src/auth/PrivateRoute.tsx

@@ -0,0 +1,33 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React, {
+  FC,
+} from 'react';
+import { Route, RouteProps } from 'react-router-dom';
+
+import Login from 'views/Login';
+import { useAuthContext } from './context';
+
+const PrivateRoute: FC<RouteProps> = (props) => {
+  const { hasValidAuthToken } = useAuthContext();
+  return hasValidAuthToken ? <Route {...props} /> : <Login />;
+};
+
+export default PrivateRoute;