diff --git a/nifi-huawei-nar/pom.xml b/nifi-huawei-nar/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..4f4300d46692fddc339d3a741108dbd956cbc08b --- /dev/null +++ b/nifi-huawei-nar/pom.xml @@ -0,0 +1,49 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-huawei-bundle + 1.18.0 + + + nifi-huawei-nar + nar + + true + true + + + + + org.apache.nifi + nifi-huawei-service-api-nar + 1.18.0 + nar + + + org.apache.nifi + nifi-huawei-processors + 1.18.0 + + + org.slf4j + jcl-over-slf4j + + + diff --git a/nifi-huawei-nar/src/main/resources/META-INF/LICENSE b/nifi-huawei-nar/src/main/resources/META-INF/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..293a59c46bf56c8a2718e23ff1fe632191bc320e --- /dev/null +++ b/nifi-huawei-nar/src/main/resources/META-INF/LICENSE @@ -0,0 +1,232 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +APACHE NIFI SUBCOMPONENTS: + +The Apache NiFi project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + + The binary distribution of this product bundles 'Bouncy Castle JDK 1.5' + under an MIT style license. + + Copyright (c) 2000 - 2015 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. diff --git a/nifi-huawei-nar/src/main/resources/META-INF/NOTICE b/nifi-huawei-nar/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..c4ff9a52e3021669e45df6f5c5b6f85b42136d84 --- /dev/null +++ b/nifi-huawei-nar/src/main/resources/META-INF/NOTICE @@ -0,0 +1,122 @@ +nifi-huawei-nar +Copyright 2015-2020 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +****************** +Apache Software License v2 +****************** + +The following binary components are provided under the Apache Software License v2 + + (ASLv2) Apache HttpComponents + The following NOTICE information applies: + Apache HttpClient + Copyright 1999-2014 The Apache Software Foundation + + Apache HttpCore + Copyright 2005-2014 The Apache Software Foundation + + This project contains annotations derived from JCIP-ANNOTATIONS + Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net + + (ASLv2) Joda Time + The following NOTICE information applies: + This product includes software developed by + Joda.org (http://www.joda.org/). + + (ASLv2) Apache Commons Codec + The following NOTICE information applies: + Apache Commons Codec + Copyright 2002-2014 The Apache Software Foundation + + src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java + contains test data from http://aspell.net/test/orig/batch0.tab. + Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + + =============================================================================== + + The content of package org.apache.commons.codec.language.bm has been translated + from the original php source code available at http://stevemorse.org/phoneticinfo.htm + with permission from the original authors. + Original source copyright: + Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + + (ASLv2) Apache Commons Lang + The following NOTICE information applies: + Apache Commons Lang + Copyright 2001-2014 The Apache Software Foundation + + This product includes software from the Spring Framework, + under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + + (ASLv2) Apache Commons BeanUtils + The following NOTICE information applies: + Apache Commons BeanUtils + Copyright 2000-2016 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + (ASLv2) Amazon Web Services SDK + The following NOTICE information applies: + Copyright 2010-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. + + This product includes software developed by + Amazon Technologies, Inc (http://www.amazon.com/). + + ********************** + THIRD PARTY COMPONENTS + ********************** + This software includes third party software subject to the following copyrights: + - XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. + - PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. + + (ASLv2) Jackson JSON processor + The following NOTICE information applies: + # Jackson JSON processor + + Jackson is a high-performance, Free/Open Source JSON processing library. + It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has + been in development since 2007. + It is currently developed by a community of developers, as well as supported + commercially by FasterXML.com. + + ## Licensing + + Jackson core and extension components may licensed under different licenses. + To find the details that apply to this artifact see the accompanying LICENSE file. + For more information, including possible other licensing options, contact + FasterXML.com (http://fasterxml.com). + + ## Credits + + A list of contributors may be found from CREDITS file, which is included + in some artifacts (usually source distributions); but is always available + from the source code management (SCM) system project uses. + + + (ASLv2) This includes derived works from apigateway-generic-java-sdk project (https://github.com/rpgreen/apigateway-generic-java-sdk) + https://github.com/rpgreen/apigateway-generic-java-sdk/commit/32eea44cc855a530c9b4a28b9f3601a41bc85618 as the point reference: + The derived work is adapted from + main/ca/ryangreen/apigateway/generic/ + GenericApiGatewayClient.java + GenericApiGatewayClientBuilder.java + GenericApiGatewayException.java + GenericApiGatewayRequest.java + GenericApiGatewayRequestBuilder.java + test/ca/ryangreen/apigateway/generic/ + GenericApiGatewayClientTest.java + LambdaMatcher.java + and can be found in the directories: + nifi-nar-bundles/nifi-aws-bundle/nifi-aws-abstract-processors/src/main/../wag/client/ + GenericApiGatewayClient.java + GenericApiGatewayClientBuilder.java + GenericApiGatewayException.java + GenericApiGatewayRequest.java + GenericApiGatewayRequestBuilder.java + Validate.java + nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/../wag/ + RequestMatcher.java + GetAWSGatewayApiTest.java \ No newline at end of file diff --git a/nifi-huawei-processors/pom.xml b/nifi-huawei-processors/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..35c76d3f75bcc9a0514b92bb7aaa19d81389fab9 --- /dev/null +++ b/nifi-huawei-processors/pom.xml @@ -0,0 +1,126 @@ + + + + 4.0.0 + + org.apache.nifi + nifi-huawei-bundle + 1.18.0 + + + nifi-huawei-processors + jar + + + + org.apache.nifi + nifi-api + + + org.apache.nifi + nifi-distributed-cache-client-service-api + + + org.apache.nifi + nifi-listed-entity + 1.18.0 + + + org.apache.nifi + nifi-huawei-service-api + 1.18.0 + provided + + + org.apache.nifi + nifi-record-serialization-service-api + + + org.apache.nifi + nifi-record + + + org.apache.nifi + nifi-standard-record-utils + 1.18.0 + + + org.apache.nifi + nifi-ssl-context-service-api + + + com.huaweicloud + esdk-obs-java-bundle + + + com.huaweicloud.sdk + huaweicloud-sdk-smn + 3.1.33 + + + com.huaweicloud.sdk + huaweicloud-sdk-dli + 3.1.33 + + + commons-beanutils + commons-beanutils + 1.9.4 + + + commons-logging + commons-logging + + + + + org.slf4j + jcl-over-slf4j + + + org.apache.nifi + nifi-mock + 1.18.0 + test + + + com.squareup.okhttp3 + mockwebserver + test + + + org.apache.nifi + nifi-mock-record-utils + test + + + org.apache.nifi + nifi-record-serialization-services + 1.18.0 + test + + + org.apache.nifi + nifi-schema-registry-service-api + 1.18.0 + provided + + + org.apache.nifi + nifi-proxy-configuration-api + + + diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/abstractprocessor/AbstractDLIProcessor.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/abstractprocessor/AbstractDLIProcessor.java new file mode 100644 index 0000000000000000000000000000000000000000..99983ac2513d751f1253de2b0162d8beb598dc59 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/abstractprocessor/AbstractDLIProcessor.java @@ -0,0 +1,76 @@ +package org.apache.nifi.processors.huawei.abstractprocessor; + +import com.huaweicloud.sdk.dli.v1.*; + +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.ProcessSessionFactory; +import org.apache.nifi.processor.VerifiableProcessor; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processors.huawei.dli.DLIUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public abstract class AbstractDLIProcessor extends AbstractHuaweiProcessor implements VerifiableProcessor { + + protected volatile DliClient dliClient; + + @OnScheduled + public void onScheduled(final ProcessContext context) { + + dliClient = DLIUtils.createClient(context); + } + + @Override + public List verify(ProcessContext context, ComponentLog verificationLogger, Map attributes) { + + final List results = new ArrayList<>(); + + try { + + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL) + .verificationStepName("Create Client and Configure Region") + .explanation("Successfully created DLI Client and configured Region") + .build()); + + } catch (final Exception e) { + + verificationLogger.error("Failed to create DLI Client", e); + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.FAILED) + .verificationStepName("Create Client and Configure Region") + .explanation("Failed to crete DLI Client or configure Region: " + e.getMessage()) + .build()); + } + + return results; + } + + /* + * Allow optional override of onTrigger with the ProcessSessionFactory where required for huawei processors + * + * @see AbstractProcessor + */ + @Override + public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException { + final ProcessSession session = sessionFactory.createSession(); + try { + onTrigger(context, session); + session.commitAsync(); + } catch (final Throwable t) { + session.rollback(true); + throw t; + } + } + + /* + * Default to requiring the "standard" onTrigger with a single ProcessSession + */ + public abstract void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException; +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/abstractprocessor/AbstractHuaweiProcessor.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/abstractprocessor/AbstractHuaweiProcessor.java new file mode 100644 index 0000000000000000000000000000000000000000..9d00cccfe301694c07bbbd9af29c24ae2f152026 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/abstractprocessor/AbstractHuaweiProcessor.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.abstractprocessor; + +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.processor.*; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.proxy.ProxyConfiguration; +import org.apache.nifi.proxy.ProxyConfigurationService; +import org.apache.nifi.proxy.ProxySpec; + +import java.util.*; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; + +/** + * Abstract base class for huawei processors. This class uses huawei credentials for creating obs clients + */ +public abstract class AbstractHuaweiProcessor extends AbstractSessionFactoryProcessor { + + public static final Relationship REL_SUCCESS = new Relationship.Builder().name("success") + .description("FlowFiles are routed to success relationship").build(); + public static final Relationship REL_FAILURE = new Relationship.Builder().name("failure") + .description("FlowFiles are routed to failure relationship").build(); + + public static final Set relationships = Set.of(REL_SUCCESS, REL_FAILURE); + + private static final ProxySpec[] PROXY_SPECS = {ProxySpec.HTTP_AUTH}; + + @Override + public Set getRelationships() { + return relationships; + } + + @Override + protected Collection customValidate(final ValidationContext validationContext) { + final List validationResults = new ArrayList<>(super.customValidate(validationContext)); + + final boolean accessKeySet = validationContext.getProperty(ACCESS_KEY).isSet(); + final boolean secretKeySet = validationContext.getProperty(SECRET_KEY).isSet(); + final boolean credentialsSet = validationContext.getProperty(HUAWEI_CREDENTIALS_PROVIDER_SERVICE).isSet(); + if (!(credentialsSet || (accessKeySet && secretKeySet))) { + validationResults.add(new ValidationResult.Builder().input("Ak/sk/credentials").valid(false).explanation("credentials and one of the Ak/SK entries must be set").build()); + } + + final boolean proxyHostSet = validationContext.getProperty(PROXY_HOST).isSet(); + final boolean proxyPortSet = validationContext.getProperty(PROXY_HOST_PORT).isSet(); + final boolean proxyConfigServiceSet = validationContext.getProperty(ProxyConfigurationService.PROXY_CONFIGURATION_SERVICE).isSet(); + + if ((proxyHostSet && !proxyPortSet) || (!proxyHostSet && proxyPortSet)) { + validationResults.add(new ValidationResult.Builder().subject("Proxy Host and Port").valid(false).explanation("If Proxy Host or Proxy Port is set, both must be set").build()); + } + + final boolean proxyUserSet = validationContext.getProperty(PROXY_USERNAME).isSet(); + final boolean proxyPwdSet = validationContext.getProperty(PROXY_PASSWORD).isSet(); + + if ((proxyUserSet && !proxyPwdSet) || (!proxyUserSet && proxyPwdSet)) { + validationResults.add(new ValidationResult.Builder().subject("Proxy User and Password").valid(false).explanation("If Proxy Username or Proxy Password is set, both must be set").build()); + } + + if (proxyUserSet && !proxyHostSet) { + validationResults.add(new ValidationResult.Builder().subject("Proxy").valid(false).explanation("If Proxy Username or Proxy Password").build()); + } + + ProxyConfiguration.validateProxySpec(validationContext, validationResults, PROXY_SPECS); + + if (proxyHostSet && proxyConfigServiceSet) { + validationResults.add(new ValidationResult.Builder().subject("Proxy Configuration Service").valid(false) + .explanation("Either Proxy Username and Proxy Password must be set or Proxy Configuration Service but not both").build()); + } + + return validationResults; + } + + /* + * Allow optional override of onTrigger with the ProcessSessionFactory where required for huawei processors + * + * @see AbstractProcessor + */ + @Override + public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException { + final ProcessSession session = sessionFactory.createSession(); + try { + onTrigger(context, session); + session.commitAsync(); + } catch (final Throwable t) { + session.rollback(true); + throw t; + } + } + + /* + * Default to requiring the "standard" onTrigger with a single ProcessSession + */ + public abstract void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException; +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/abstractprocessor/AbstractOBSProcessor.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/abstractprocessor/AbstractOBSProcessor.java new file mode 100644 index 0000000000000000000000000000000000000000..ab10ba0a163f98b3f41299f9772974ad3ef7e06a --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/abstractprocessor/AbstractOBSProcessor.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.abstractprocessor; + +import com.obs.services.ObsClient; +import com.obs.services.exception.ObsException; +import com.obs.services.model.*; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.VerifiableProcessor; +import org.apache.nifi.processors.huawei.obs.OBSRegions; +import org.apache.nifi.processors.huawei.obs.OBSUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; +import static org.apache.nifi.processors.huawei.obs.Constants.*; + +public abstract class AbstractOBSProcessor extends AbstractHuaweiProcessor implements VerifiableProcessor { + protected volatile ObsClient client; + @OnScheduled + public void onScheduled(final ProcessContext context) { + client = OBSUtils.createClient(context, getLogger()); + } + + @Override + public List verify(final ProcessContext context, final ComponentLog verificationLogger, final Map attributes) { + final List results = new ArrayList<>(); + + try { + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL) + .verificationStepName("Create Client and Configure Region") + .explanation("Successfully created OBS Client and configured Region") + .build()); + } catch (final Exception e) { + verificationLogger.error("Failed to create OBS Client", e); + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.FAILED) + .verificationStepName("Create Client and Configure Region") + .explanation("Failed to crete OBS Client or configure Region: " + e.getMessage()) + .build()); + } + return results; + } + + protected GranteeInterface createGrantee(final String value) { + if (StringUtils.isBlank(value)) { + return null; + } + return new CanonicalGrantee(value); + } + + protected final List createGrantees(final String value) { + if (StringUtils.isBlank(value)) { + return Collections.emptyList(); + } + + final List grantees = new ArrayList<>(); + final String[] values = value.split(","); + for (final String val : values) { + final String identifier = val.trim(); + final GranteeInterface grantee = createGrantee(identifier); + if (grantee != null) { + grantees.add(grantee); + } + } + return grantees; + } + + /** + * Create AccessControlList if appropriate properties are configured. + * + * @param context ProcessContext + * @param flowFile FlowFile + * @return AccessControlList or null if no ACL properties were specified + */ + protected final AccessControlList createACL(final ProcessContext context, final FlowFile flowFile) { + AccessControlList acl = null; + + final String ownerId = context.getProperty(OWNER).evaluateAttributeExpressions(flowFile).getValue(); + if (StringUtils.isNotBlank(ownerId)) { + final Owner owner = new Owner(); + owner.setId(ownerId); + if (acl == null) { + acl = new AccessControlList(); + } + acl.setOwner(owner); + acl.grantPermission(new CanonicalGrantee(ownerId), Permission.PERMISSION_FULL_CONTROL); + } + + for (final GranteeInterface grantee : createGrantees(context.getProperty(FULL_CONTROL_USER_LIST).evaluateAttributeExpressions(flowFile).getValue())) { + if (acl == null) { + acl = new AccessControlList(); + } + acl.grantPermission(grantee, Permission.PERMISSION_FULL_CONTROL); + } + + for (final GranteeInterface grantee : createGrantees(context.getProperty(READ_USER_LIST).evaluateAttributeExpressions(flowFile).getValue())) { + if (acl == null) { + acl = new AccessControlList(); + } + acl.grantPermission(grantee, Permission.PERMISSION_READ); + } + + for (final GranteeInterface grantee : createGrantees(context.getProperty(READ_ACL_LIST).evaluateAttributeExpressions(flowFile).getValue())) { + if (acl == null) { + acl = new AccessControlList(); + } + acl.grantPermission(grantee, Permission.PERMISSION_READ_ACP); + } + + for (final GranteeInterface grantee : createGrantees(context.getProperty(WRITE_ACL_LIST).evaluateAttributeExpressions(flowFile).getValue())) { + if (acl == null) { + acl = new AccessControlList(); + } + acl.grantPermission(grantee, Permission.PERMISSION_WRITE_ACP); + } + + return acl; + } + + protected FlowFile extractExceptionDetails(final Exception e, final ProcessSession session, FlowFile flowFile) { + flowFile = session.putAttribute(flowFile, OBS_EXCEPTION, e.getClass().getName()); + if (e instanceof ObsException) { + flowFile = putAttribute(session, flowFile, OBS_ERROR_CODE, ((ObsException) e).getErrorCode()); + flowFile = putAttribute(session, flowFile, OBS_ERROR_Message, ((ObsException) e).getErrorMessage()); + flowFile = putAttribute(session, flowFile, OBS_ADDITIONAL_DETAILS, ((ObsException) e).getXmlMessage()); + return flowFile; + + } + return putAttribute(session, flowFile, OBS_ERROR_Message, e.getMessage()); + } + + private FlowFile putAttribute(final ProcessSession session, final FlowFile flowFile, final String key, final Object value) { + return (value == null) ? flowFile : session.putAttribute(flowFile, key, value.toString()); + } + + /** + * Create CannedAccessControlList if CANNED_ACL property specified. + * + * @param context ProcessContext + * @param flowFile FlowFile + * @return CannedAccessControlList or null if not specified + */ + protected final AccessControlList createCannedACL(final ProcessContext context, final FlowFile flowFile) { + final String cannedAclString = context.getProperty(CANNED_ACL).evaluateAttributeExpressions(flowFile).getValue(); + if (StringUtils.isNoneBlank(cannedAclString)) { + switch (cannedAclString) { + case "Private": + return AccessControlList.REST_CANNED_PRIVATE; + case "PublicRead": + return AccessControlList.REST_CANNED_PUBLIC_READ; + case "PublicReadWrite": + return AccessControlList.REST_CANNED_PUBLIC_READ_WRITE; + case "PublicReadWriteDelivered": + return AccessControlList.REST_CANNED_PUBLIC_READ_WRITE_DELIVERED; + default: + } + } + return null; + } + + + + + public ObsClient getClient() { + return client; + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/abstractprocessor/AbstractSMNProcessor.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/abstractprocessor/AbstractSMNProcessor.java new file mode 100644 index 0000000000000000000000000000000000000000..15da11de5c4c3fee64bdbba3d4a0e35d4079ef40 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/abstractprocessor/AbstractSMNProcessor.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.abstractprocessor; + +import com.huaweicloud.sdk.smn.v2.SmnClient; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.*; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processors.huawei.smn.SMNUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public abstract class AbstractSMNProcessor extends AbstractHuaweiProcessor implements VerifiableProcessor { + + protected volatile SmnClient smnClient; + + @OnScheduled + public void onScheduled(final ProcessContext context) { + + smnClient = SMNUtils.createClient(context); + } + + @Override + public List verify(final ProcessContext processContext, final ComponentLog componentLog, final Map map) { + + final List results = new ArrayList<>(); + + try { + + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL) + .verificationStepName("Create Client and Configure Region") + .explanation("Successfully created SMN Client and configured Region") + .build()); + + } catch (final Exception e) { + + componentLog.error("Failed to create SMN Client", e); + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.FAILED) + .verificationStepName("Create Client and Configure Region") + .explanation("Failed to crete SMN Client or configure Region: " + e.getMessage()) + .build()); + } + + return results; + } + + public SmnClient getSmnClient() { return smnClient; } + + /* + * Allow optional override of onTrigger with the ProcessSessionFactory where required for huawei processors + * + * @see AbstractProcessor + */ + @Override + public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException { + final ProcessSession session = sessionFactory.createSession(); + try { + onTrigger(context, session); + session.commitAsync(); + } catch (final Throwable t) { + session.rollback(true); + throw t; + } + } + + /* + * Default to requiring the "standard" onTrigger with a single ProcessSession + */ + public abstract void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException; +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/common/PropertyDescriptors.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/common/PropertyDescriptors.java new file mode 100644 index 0000000000000000000000000000000000000000..7998ce863123b51f41bb339e22b32279c585f011 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/common/PropertyDescriptors.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.common; + +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.huawei.dli.DLIUtils; +import org.apache.nifi.processors.huawei.obs.OBSRegions; +import org.apache.nifi.processors.huawei.credentials.provider.service.HuaweiCredentialsProviderService; +import org.apache.nifi.processors.huawei.obs.Constants; +import org.apache.nifi.processors.huawei.obs.ObsServiceEncryptionService; +import org.apache.nifi.processors.huawei.smn.SMNUtils; +import org.apache.nifi.proxy.ProxyConfiguration; +import org.apache.nifi.ssl.SSLContextService; + +import java.util.stream.Collectors; + +public interface PropertyDescriptors { + PropertyDescriptor ACCESS_KEY = new PropertyDescriptor.Builder() + .name("Access Key") + .displayName("Access Key ID") + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .sensitive(true) + .build(); + + PropertyDescriptor SECRET_KEY = new PropertyDescriptor.Builder() + .name("Secret Key") + .displayName("Secret Access Key") + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .sensitive(true) + .build(); + PropertyDescriptor PROXY_HOST = new PropertyDescriptor.Builder() + .name("Proxy Host") + .description("Proxy host name or IP") + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + PropertyDescriptor PROXY_HOST_PORT = new PropertyDescriptor.Builder() + .name("Proxy Host Port") + .description("Proxy host port") + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .required(false) + .addValidator(StandardValidators.PORT_VALIDATOR) + .build(); + + PropertyDescriptor PROXY_USERNAME = new PropertyDescriptor.Builder() + .name("proxy-user-name") + .displayName("Proxy Username") + .description("Proxy username") + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + PropertyDescriptor PROXY_PASSWORD = new PropertyDescriptor.Builder() + .name("proxy-user-password") + .displayName("Proxy Password") + .description("Proxy password") + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .sensitive(true) + .build(); + + PropertyDescriptor OBS_REGION = new PropertyDescriptor.Builder() + .name("OBS Region") + .description("If the destination Region is Not found, select Not Found and enter the Endpoint address of the destination Region in the Endpoint Override URL box") + .required(true) + .allowableValues(OBSRegions.getAvailableOBSRegions()) + .defaultValue(OBSRegions.DEFAULT_REGION.getName()) + .build(); + PropertyDescriptor ENDPOINT_OVERRIDE_URL = new PropertyDescriptor.Builder() + .name("Endpoint Override URL") + .description("The endpoint specified in this option box takes effect only when the OBS Region option is Not found") + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() + .name("Communications Timeout") + .required(true) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .defaultValue("30 secs") + .build(); + + PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("SSL Context Service") + .description("Specifies an optional SSL Context Service that, if provided, will be used to create connections") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + /** + * Huawei credentials provider service + */ + PropertyDescriptor HUAWEI_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("Huawei Credentials Provider service") + .displayName("Huawei Credentials Provider Service") + .description("The Controller Service that is used to obtain huawei credentials provider") + .required(false) + .identifiesControllerService(HuaweiCredentialsProviderService.class) + .build(); + PropertyDescriptor DELIMITER = new PropertyDescriptor.Builder() + .name("delimiter") + .displayName("Delimiter") + .expressionLanguageSupported(ExpressionLanguageScope.NONE) + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .description("Specifies the character used to group the object name. " + + "The grouping steps are as follows: " + + "① First select all object names that contain this Delimiter; " + + "② Remove the prefix field from the object names. (If the prefix field is not specified in the request, skip this step.) " + + "③ Group by the string between the first character and Delimiter (later returned as CommonPrefix).") + .build(); + + PropertyDescriptor PREFIX = new PropertyDescriptor.Builder() + .name("prefix") + .displayName("Prefix") + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .description("The prefix used to filter the object list. In most cases, it should end with a forward slash ('/').") + .build(); + PropertyDescriptor WRITE_USER_METADATA = new PropertyDescriptor.Builder() + .name("write-obs-user-metadata") + .displayName("Write User Metadata") + .description("If set to 'True', the user defined metadata associated with the OBS object will be added to FlowFile attributes/records") + .required(true) + .allowableValues(new AllowableValue("true", "True"), new AllowableValue("false", "False")) + .defaultValue("false") + .build(); + PropertyDescriptor FULL_CONTROL_USER_LIST = new PropertyDescriptor.Builder() + .name("FullControl User List") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .description("A comma-separated list of huaweiCloud account ID's that specifies who should have Full Control for an object") + .defaultValue("${obs.permissions.full.users}") + .build(); + + PropertyDescriptor READ_USER_LIST = new PropertyDescriptor.Builder() + .name("Read Permission User List") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .description("A comma-separated list of huaweiCloud account ID's that specifies who should have Read Access for an object") + .defaultValue("${obs.permissions.read.users}") + .build(); + PropertyDescriptor READ_ACL_LIST = new PropertyDescriptor.Builder() + .name("Read ACL User List") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .description("A comma-separated list of huaweiCloud account ID's that specifies who should have permissions to read the Access Control List for an object") + .defaultValue("${obs.permissions.readacl.users}") + .build(); + PropertyDescriptor WRITE_ACL_LIST = new PropertyDescriptor.Builder() + .name("Write ACL User List") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .description("A comma-separated list of huaweiCloud account ID's that specifies who should have permissions to change the Access Control List for an object") + .defaultValue("${obs.permissions.writeacl.users}") + .build(); + PropertyDescriptor CANNED_ACL = new PropertyDescriptor.Builder() + .name("canned-acl") + .displayName("Canned ACL") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .description("OBS Canned ACL for an object, one of: Private, PublicRead, PublicReadWrite, PublicReadWriteDelivered; " + + "will be ignored if any other ACL/permission/owner property is specified; details: https://support.huaweicloud.com/sdk-java-devg-obs/obs_21_0406.html") + .defaultValue("${obs.permissions.cannedacl}") + .build(); + + PropertyDescriptor OWNER = new PropertyDescriptor.Builder() + .name("Owner") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .description("The huaweiCloud Account ID to use for the object's owner") + .defaultValue("${obs.owner}") + .build(); + PropertyDescriptor BUCKET = new PropertyDescriptor.Builder() + .name("Bucket") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + PropertyDescriptor KEY = new PropertyDescriptor.Builder() + .name("Object Key") + .required(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .defaultValue("${filename}") + .build(); + PropertyDescriptor ENCRYPTION_SERVICE = new PropertyDescriptor.Builder() + .name("encryption-service") + .displayName("Encryption Service") + .description("Specifies the Encryption Service Controller used to configure requests. " + + "PutOBSObject/FetchOBSObject: Only needs to be configured in case of Server-side SSE_KMS/SSE_C encryption") + .required(false) + .identifiesControllerService(ObsServiceEncryptionService.class) + .build(); + PropertyDescriptor PROXY_CONFIGURATION_SERVICE = ProxyConfiguration.createProxyConfigPropertyDescriptor(false, Constants.PROXY_SPECS); + PropertyDescriptor SMN_REGION = new PropertyDescriptor.Builder() + .name("SMN Region") + .description("If the destination Region is Not found, select Not Found and enter the Endpoint address of the destination Region in the Endpoint Override URL box") + .required(true) + .allowableValues(SMNUtils.getAvailableRegions().stream() + .map(x -> new AllowableValue(x.getId(), x.getId(), x.getId())).collect(Collectors.toList()) + .toArray(new AllowableValue[0])) + .build(); + PropertyDescriptor DLI_REGION = new PropertyDescriptor.Builder() + .name("DLI Region") + .description("Indicates the region where the API is currently available.") + .required(true) + .allowableValues(DLIUtils.getAvailableRegions().stream() + .map(x -> new AllowableValue(x.getId(), x.getId(), x.getId())).collect(Collectors.toList()) + .toArray(new AllowableValue[0])) + .build(); +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/factory/CredentialsProviderFactory.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/factory/CredentialsProviderFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..46ec013da2e70d07422e710766ee72150151a337 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/factory/CredentialsProviderFactory.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.credentials.provider.factory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; + +import com.obs.services.IObsCredentialsProvider; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.processors.huawei.credentials.provider.factory.strategies.AccessKeyPairCredentialsStrategy; + + +/** + * Generates HuaweiCloud credentials in the form of HuaweiCredentialsProvider implementations for processors + * and controller services. The factory supports a number of strategies for specifying and validating + * HuaweiCloud credentials, interpreted as an ordered list of most-preferred to least-preferred. It also supports + * derived credential strategies like Assume Role, which require a primary credential as an input. + * + * Additional strategies should implement CredentialsStrategy, then be added to the strategies list in the + * constructor. + * + * @see org.apache.nifi.processors.huawei.credentials.provider.factory.strategies + */ +public class CredentialsProviderFactory { + + /** + * Validates HuaweiCloud credential properties against the configured strategies to report any validation errors. + * @return Validation errors + */ + public Collection validate(final ValidationContext validationContext) { + final CredentialsStrategy selectedStrategy = new AccessKeyPairCredentialsStrategy(); + final ArrayList validationFailureResults = new ArrayList(); + final Collection strategyValidationFailures = selectedStrategy.validate(validationContext); + if (strategyValidationFailures != null) { + validationFailureResults.addAll(strategyValidationFailures); + } + return validationFailureResults; + } + + /** + * Produces the HuaweiCredentialsProvider according to the given property set and the strategies configured in + * the factory. + * @return HuaweiCredentialsProvider implementation + */ + public IObsCredentialsProvider getCredentialsProvider(final Map properties) { + final CredentialsStrategy primaryStrategy = new AccessKeyPairCredentialsStrategy(); + return primaryStrategy.getCredentialsProvider(properties); + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/factory/CredentialsStrategy.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/factory/CredentialsStrategy.java new file mode 100644 index 0000000000000000000000000000000000000000..410407460ae52cc58c4b7f8256e7034c5b7f7644 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/factory/CredentialsStrategy.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.credentials.provider.factory; + +import java.util.Collection; +import java.util.Map; + +import com.obs.services.IObsCredentialsProvider; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; + + +/** + * Specifies a strategy for validating and creating HuaweiCloud credentials from a list of properties configured on a + * Processor, Controller Service, Reporting Service, or other component. Supports both primary credentials like + * default credentials or API keys and also derived credentials from Assume Role. + */ +public interface CredentialsStrategy { + + /** + * Name of the strategy, suitable for displaying to a user in validation messages. + * @return strategy name + */ + String getName(); + + /** + * Validates the properties belonging to this strategy, given the selected primary strategy. Errors may result + * from individually malformed properties, invalid combinations of properties, or inappropriate use of properties + * not consistent with the primary strategy. + * @return validation errors + */ + Collection validate(ValidationContext validationContext); + + /** + * Creates an HuaweiCredentialsProvider instance for this strategy, given the properties defined by the user. + */ + IObsCredentialsProvider getCredentialsProvider(Map properties); +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/factory/strategies/AbstractCredentialsStrategy.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/factory/strategies/AbstractCredentialsStrategy.java new file mode 100644 index 0000000000000000000000000000000000000000..08fa9f2802d124cbf2a3efdf5ab4029b9a68c837 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/factory/strategies/AbstractCredentialsStrategy.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.credentials.provider.factory.strategies; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; + +import com.obs.services.IObsCredentialsProvider; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.processors.huawei.credentials.provider.factory.CredentialsStrategy; + +/** + * Partial implementation of CredentialsStrategy to support most simple property-based strategies. + */ +public abstract class AbstractCredentialsStrategy implements CredentialsStrategy { + private final String name; + private final PropertyDescriptor[] requiredProperties; + + public AbstractCredentialsStrategy(String name, PropertyDescriptor[] requiredProperties) { + this.name = name; + this.requiredProperties = requiredProperties; + } + + @Override + public Collection validate(final ValidationContext validationContext) { + String requiredMessageFormat = "property %1$s must be set with %2$s"; + Collection validationFailureResults = null; + for (PropertyDescriptor requiredProperty : requiredProperties) { + if (!validationContext.getProperty(requiredProperty).isSet()) { + String message = String.format(requiredMessageFormat, requiredProperty.getDisplayName(), + getName()); + if (validationFailureResults == null) { + validationFailureResults = new ArrayList(); + } + validationFailureResults.add(new ValidationResult.Builder() + .subject(requiredProperty.getDisplayName()) + .valid(false) + .explanation(message).build()); + } + } + + return validationFailureResults; + } + + public abstract IObsCredentialsProvider getCredentialsProvider(Map properties); + + public String getName() { + return name; + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/factory/strategies/AccessKeyPairCredentialsStrategy.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/factory/strategies/AccessKeyPairCredentialsStrategy.java new file mode 100644 index 0000000000000000000000000000000000000000..80e5d6de95bb1a34aae625a288945130457ec197 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/factory/strategies/AccessKeyPairCredentialsStrategy.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.credentials.provider.factory.strategies; + +import java.util.Map; + +import com.obs.services.BasicObsCredentialsProvider; +import com.obs.services.IObsCredentialsProvider; +import com.obs.services.internal.security.BasicSecurityKey; +import com.obs.services.model.ISecurityKey; +import org.apache.nifi.components.PropertyDescriptor; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; + +/** + * Supports HuaweiCloud credentials defined by an Access Key and Secret Key pair. + */ +public class AccessKeyPairCredentialsStrategy extends AbstractCredentialsStrategy { + + public AccessKeyPairCredentialsStrategy() { + super("Access Key Pair", new PropertyDescriptor[] { + ACCESS_KEY, + SECRET_KEY + }); + } + + @Override + public IObsCredentialsProvider getCredentialsProvider(Map properties) { + String accessKey = properties.get(ACCESS_KEY); + String secretKey = properties.get(SECRET_KEY); + ISecurityKey securityKey = new BasicSecurityKey(accessKey, secretKey); + return new BasicObsCredentialsProvider(securityKey); + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/service/HuaweiCredentialsProviderControllerService.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/service/HuaweiCredentialsProviderControllerService.java new file mode 100644 index 0000000000000000000000000000000000000000..2c41374d532248caddda75a5ad2b47d4b60b96aa --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/service/HuaweiCredentialsProviderControllerService.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.credentials.provider.service; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.obs.services.IObsCredentialsProvider; +import org.apache.nifi.annotation.behavior.Restricted; +import org.apache.nifi.annotation.behavior.Restriction; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnEnabled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.RequiredPermission; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processors.huawei.credentials.provider.factory.CredentialsProviderFactory; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.ACCESS_KEY; +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.SECRET_KEY; + + +/** + * Implementation of HuaweiCredentialsProviderService interface + * + * @see HuaweiCredentialsProviderService + */ +@CapabilityDescription("Defines credentials for Huawei Web Services processors. " + + "Uses default credentials without configuration. " + + "Default credentials support ECS instance profile/role, default user profile, environment variables, etc. " + + "Additional options include access key / secret key pairs, credentials file, named profile, and assume role credentials.") +@Tags({ "huawei", "credentials","provider" }) +@Restricted( + restrictions = { + @Restriction( + requiredPermission = RequiredPermission.ACCESS_ENVIRONMENT_CREDENTIALS, + explanation = "The default configuration can read environment variables and system properties for credentials" + ) + } +) +public class HuaweiCredentialsProviderControllerService extends AbstractControllerService implements HuaweiCredentialsProviderService { + private volatile IObsCredentialsProvider credentialsProvider; + protected final CredentialsProviderFactory credentialsProviderFactory = new CredentialsProviderFactory(); + + @Override + protected List getSupportedPropertyDescriptors() { + final List props = new ArrayList<>(); + props.add(ACCESS_KEY); + props.add(SECRET_KEY); + return Collections.unmodifiableList(props); + } + + @Override + public IObsCredentialsProvider getCredentialsProvider() throws ProcessException { + return credentialsProvider; + } + + @Override + protected Collection customValidate(final ValidationContext validationContext) { + return credentialsProviderFactory.validate(validationContext); + } + + @OnEnabled + public void onConfigured(final ConfigurationContext context) { + final Map evaluatedProperties = new HashMap<>(context.getProperties()); + evaluatedProperties.keySet().forEach(propertyDescriptor -> { + if (propertyDescriptor.isExpressionLanguageSupported()) { + evaluatedProperties.put(propertyDescriptor, + context.getProperty(propertyDescriptor).evaluateAttributeExpressions().getValue()); + } + }); + credentialsProvider = credentialsProviderFactory.getCredentialsProvider(evaluatedProperties); + getLogger().debug("Using credentials provider: " + credentialsProvider.getClass()); + } + + @Override + public String toString() { + return "HuaweiCredentialsProviderService[id=" + getIdentifier() + "]"; + } +} \ No newline at end of file diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/dli/DLICreateSqlJob.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/dli/DLICreateSqlJob.java new file mode 100644 index 0000000000000000000000000000000000000000..166cc064d143556262bd07352564c2fbf193a961 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/dli/DLICreateSqlJob.java @@ -0,0 +1,231 @@ +package org.apache.nifi.processors.huawei.dli; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.huaweicloud.sdk.dli.v1.model.*; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.huawei.abstractprocessor.AbstractDLIProcessor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; + +@SupportsBatching +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@Tags({"HuaweiCloud", "DLI", "CreateSqlJob"}) +@CapabilityDescription("This API is used to submit jobs to the queue by executing SQL statements. Jobs include the following types: DDL, DCL, IMPORT, QUERY, and INSERT. " + + "Among them, the function of IMPORT is the same as that of importing data, the difference is only in the way of implementation.") +public class DLICreateSqlJob extends AbstractDLIProcessor { + + public static final PropertyDescriptor DLI_PROJECT_ID = new PropertyDescriptor.Builder() + .name("project_id") + .displayName("Project ID") + .description("Project ID") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .required(true) + .build(); + + public static final PropertyDescriptor DLI_SQL_QUERY = new PropertyDescriptor.Builder() + .name("sql") + .displayName("SQL Query") + .description("The SQL statement to be executed.") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .required(true) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .build(); + + public static final PropertyDescriptor DLI_CURRENT_DB = new PropertyDescriptor.Builder() + .name("currentdb") + .displayName("Current DB") + .description("The database where the SQL statement is executed.") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .required(false) + .build(); + + public static final PropertyDescriptor DLI_QUEUE_NAME = new PropertyDescriptor.Builder() + .displayName("Queue Name") + .description("The queue name of the job to be submitted.") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .required(false) .name("queue_name") + + .build(); + + public static final PropertyDescriptor DLI_CONF = new PropertyDescriptor.Builder() + .name("conf") + .displayName("Configurations") + .description("The configuration parameters for this job in the form of \"key=value\" in an array.") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .required(false) + .build(); + + public static final PropertyDescriptor DLI_TAGS = new PropertyDescriptor.Builder() + .name("tags") + .displayName("Tags") + .description("Tags for the job in the form of key, value map in an array.") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .required(false) + .build(); + + public static final PropertyDescriptor JOB_STATUS_CHECK_INTERVAL = new PropertyDescriptor.Builder() + .name("jobStatusCheckInterval") + .displayName("Job Status Check Interval") + .description("Amount of time that get spends between status check in milliseconds.") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .defaultValue("2000") + .required(false) + .build(); + + public static final PropertyDescriptor JOB_STATUS_CHECK_ENABLE = new PropertyDescriptor.Builder() + .name("jobStatusCheckEnable") + .displayName("Job Status Check Enable") + .description("Whether the user wants to check job status or not.") + .allowableValues("true", "false") + .defaultValue("false") + .build(); + + private static final List properties = List.of(ACCESS_KEY, SECRET_KEY, DLI_PROJECT_ID, DLI_SQL_QUERY, DLI_CURRENT_DB, + DLI_QUEUE_NAME, DLI_CONF, DLI_TAGS, DLI_REGION, JOB_STATUS_CHECK_INTERVAL, JOB_STATUS_CHECK_ENABLE); + + @Override + public List getSupportedPropertyDescriptors() { + return properties; + } + + private CreateSqlJobResponse createSqlJob(CreateSqlJobRequest request) { + + return dliClient.createSqlJob(request); + } + + private ShowSqlJobStatusResponse showSqlJobStatus(ShowSqlJobStatusRequest request) { + + return dliClient.showSqlJobStatus(request); + } + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + + FlowFile flowFile = session.get(); + if (flowFile == null) { + + return; + } + + try { + + String sqlQuery = context.getProperty(DLI_SQL_QUERY).evaluateAttributeExpressions(flowFile).getValue(); + String currentDb = context.getProperty(DLI_CURRENT_DB).evaluateAttributeExpressions(flowFile).getValue(); + String queueName = context.getProperty(DLI_QUEUE_NAME).evaluateAttributeExpressions(flowFile).getValue(); + String configurations = context.getProperty(DLI_CONF).evaluateAttributeExpressions(flowFile).getValue(); + String tags = context.getProperty(DLI_TAGS).evaluateAttributeExpressions(flowFile).getValue(); + String jobStatusCheckInterval = context.getProperty(JOB_STATUS_CHECK_INTERVAL).evaluateAttributeExpressions(flowFile).getValue(); + String isJobStatusCheckEnabled = context.getProperty(JOB_STATUS_CHECK_ENABLE).evaluateAttributeExpressions(flowFile).getValue(); + + int jobStatusCheckIntervalInt = 2000; + boolean isJobStatusCheckEnabledBoolean = false; + + CreateSqlJobRequest createSqlJobRequest = new CreateSqlJobRequest(); + CreateSqlJobRequestBody body = new CreateSqlJobRequestBody(); + body.setSql(sqlQuery); + body.setCurrentdb(currentDb); + body.setQueueName(queueName); + + if (!StringUtils.isEmpty(configurations)) { + + List configurationList = new ObjectMapper().readValue(configurations, List.class); + body.setConf(configurationList); + } + + if (!StringUtils.isEmpty(tags)) { + + Map tagsMap = new ObjectMapper().readValue(tags, HashMap.class); + List tmsTagEntities = new ArrayList<>(); + + tagsMap.forEach((key, value) -> { + + TmsTagEntity tmsTagEntity = new TmsTagEntity(); + tmsTagEntity.setKey(key); + tmsTagEntity.setValue(value); + + tmsTagEntities.add(tmsTagEntity); + }); + + body.setTags(tmsTagEntities); + } + + if (!StringUtils.isEmpty(jobStatusCheckInterval)) { + + jobStatusCheckIntervalInt = Integer.parseInt(jobStatusCheckInterval); + } + + if (!StringUtils.isEmpty(isJobStatusCheckEnabled)) { + + isJobStatusCheckEnabledBoolean = Boolean.parseBoolean(isJobStatusCheckEnabled); + } + + createSqlJobRequest.withBody(body); + + CreateSqlJobResponse response = createSqlJob(createSqlJobRequest); + + ShowSqlJobStatusRequest showSqlJobStatusRequest = new ShowSqlJobStatusRequest(); + showSqlJobStatusRequest.setJobId(response.getJobId()); + + if (isJobStatusCheckEnabledBoolean) { + + boolean isJobNotFinished = true; + + while (isJobNotFinished) { + + ShowSqlJobStatusResponse showSqlJobStatusResponse = showSqlJobStatus(showSqlJobStatusRequest); + + if (showSqlJobStatusResponse.getStatus() == ShowSqlJobStatusResponse.StatusEnum.FINISHED) { + + getLogger().error("Successfully executed the sql for {}", new Object[]{flowFile}); + + session.transfer(flowFile, REL_SUCCESS); + session.getProvenanceReporter().send(flowFile, response.getJobId()); + + isJobNotFinished = false; + } + + if (showSqlJobStatusResponse.getStatus() == ShowSqlJobStatusResponse.StatusEnum.FAILED || + showSqlJobStatusResponse.getStatus() == ShowSqlJobStatusResponse.StatusEnum.CANCELLED) { + + getLogger().error("Failed to execute the sql for {}", new Object[]{flowFile}); + + session.transfer(flowFile, REL_FAILURE); + + isJobNotFinished = false; + } + + Thread.sleep(jobStatusCheckIntervalInt); + } + + } else { + + session.transfer(flowFile, REL_SUCCESS); + session.getProvenanceReporter().send(flowFile, response.getJobId()); + } + + getLogger().info("Successfully created the sql job for {}", new Object[]{flowFile}); + + } catch (Exception e) { + + getLogger().error("Failed to create the sql job for {} due to {}", new Object[]{flowFile, e}); + + flowFile = session.penalize(flowFile); + session.transfer(flowFile, REL_FAILURE); + } + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/dli/DLIUtils.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/dli/DLIUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..a779841cabe9f9479f40035b63f204a302eb0cfc --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/dli/DLIUtils.java @@ -0,0 +1,50 @@ +package org.apache.nifi.processors.huawei.dli; + +import com.huaweicloud.sdk.core.auth.BasicCredentials; +import com.huaweicloud.sdk.core.region.Region; +import com.huaweicloud.sdk.dli.v1.DliClient; +import com.huaweicloud.sdk.dli.v1.region.DliRegion; +import org.apache.nifi.processor.ProcessContext; + +import java.util.List; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; +import static org.apache.nifi.processors.huawei.dli.DLICreateSqlJob.DLI_PROJECT_ID; + +public class DLIUtils { + + public static DliClient createClient(final ProcessContext processContext) { + + final String accessKey = processContext.getProperty(ACCESS_KEY).evaluateAttributeExpressions().getValue(); + final String secretKey = processContext.getProperty(SECRET_KEY).evaluateAttributeExpressions().getValue(); + final String projectId = processContext.getProperty(DLI_PROJECT_ID).evaluateAttributeExpressions().getValue(); + final String region = processContext.getProperty(DLI_REGION).evaluateAttributeExpressions().getValue(); + + return DliClient.newBuilder().withCredential( + new BasicCredentials().withAk(accessKey).withSk(secretKey).withProjectId(projectId)) + .withRegion(DliRegion.valueOf(region)).build(); + } + + public static List getAvailableRegions() { + + return List.of( + DliRegion.AF_SOUTH_1, + DliRegion.AP_SOUTHEAST_1, + DliRegion.AP_SOUTHEAST_2, + DliRegion.AP_SOUTHEAST_3, + DliRegion.CN_EAST_2, + DliRegion.CN_EAST_3, + DliRegion.CN_NORTH_1, + DliRegion.CN_NORTH_2, + DliRegion.CN_NORTH_4, + DliRegion.CN_NORTH_9, + DliRegion.CN_SOUTH_1, + DliRegion.CN_SOUTHWEST_2, + DliRegion.LA_NORTH_2, + DliRegion.LA_SOUTH_2, + DliRegion.NA_MEXICO_1, + DliRegion.RU_NORTHWEST_2, + DliRegion.SA_BRAZIL_1 + ); + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/AbstractListHuaweiProcessor.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/AbstractListHuaweiProcessor.java new file mode 100644 index 0000000000000000000000000000000000000000..9597d06e6cf8b05e5dcb95ed1c014fffafc0a2f2 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/AbstractListHuaweiProcessor.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.huawei.obs; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.processor.DataUnit; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processor.util.list.AbstractListProcessor; +import org.apache.nifi.processor.util.list.ListableEntity; + +import java.util.concurrent.TimeUnit; + +import static org.apache.nifi.processor.util.StandardValidators.TIME_PERIOD_VALIDATOR; + +public abstract class AbstractListHuaweiProcessor extends AbstractListProcessor { + public static final PropertyDescriptor MIN_AGE = new PropertyDescriptor.Builder() + .name("Minimum File Age") + .description("The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored") + .required(true) + .addValidator(TIME_PERIOD_VALIDATOR) + .defaultValue("0 sec") + .build(); + + public static final PropertyDescriptor MAX_AGE = new PropertyDescriptor.Builder() + .name("Maximum File Age") + .description("The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored") + .required(false) + .addValidator(StandardValidators.createTimePeriodValidator(100, TimeUnit.MILLISECONDS, Long.MAX_VALUE, TimeUnit.NANOSECONDS)) + .defaultValue("100 secs") + .build(); + + public static final PropertyDescriptor MIN_SIZE = new PropertyDescriptor.Builder() + .name("Minimum File Size") + .description("The minimum size that a file must be in order to be pulled") + .required(true) + .addValidator(StandardValidators.DATA_SIZE_VALIDATOR) + .defaultValue("0 B") + .build(); + + public static final PropertyDescriptor MAX_SIZE = new PropertyDescriptor.Builder() + .name("Maximum File Size") + .description("The maximum size that a file can be in order to be pulled") + .required(false) + .addValidator(StandardValidators.DATA_SIZE_VALIDATOR) + .build(); + + protected boolean isFileInfoMatchesWithAgeAndSize(final ProcessContext context, final long minimumTimestamp, final long lastModified, final long size) { + final long minSize = context.getProperty(MIN_SIZE).asDataSize(DataUnit.B).longValue(); + final Double maxSize = context.getProperty(MAX_SIZE).asDataSize(DataUnit.B); + final long minAge = context.getProperty(MIN_AGE).asTimePeriod(TimeUnit.MILLISECONDS); + final Long maxAge = context.getProperty(MAX_AGE).asTimePeriod(TimeUnit.MILLISECONDS); + + if (lastModified < minimumTimestamp) { + return false; + } + final long fileAge = System.currentTimeMillis() - lastModified; + if (minAge > fileAge) { + return false; + } + if (maxAge != null && maxAge < fileAge) { + return false; + } + if (minSize > size) { + return false; + } + if (maxSize != null && maxSize < size) { + return false; + } + return true; + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/Constants.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/Constants.java new file mode 100644 index 0000000000000000000000000000000000000000..5ab9ed83027401d3d6df9b307c0c5884f82e8a23 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/Constants.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.huawei.obs; + +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.proxy.ProxySpec; + +public interface Constants { + String NULL_VERSION_ID = "null"; + String OBS_BUCKET = "obs.bucket"; + String OBJECT_URL = "obs.objectUrl"; + String OBS_OBJECT = "obs.objectKey"; + String FILENAME = CoreAttributes.FILENAME.key(); + String OBS_LAST_MODIFIED = "obs.lastModified"; + String OBS_OPERATION = "obs.operation"; + String OBS_CONTENT_TYPE = "obs.contentType"; + String OBS_CONTENT_DISPOSITION = "obs.contentDisposition"; + String OBS_UPLOAD_ID = "obs.uploadId"; + String OBS_VERSION = "obs.version"; + String OBS_E_TAG = "obs.eTag"; + String OBS_CACHE_CONTROL = "obs.cacheControl"; + String OBS_STORAGE_CLASS = "obs.storeClass"; + String OBS_USER_META = "obs.userMetadata"; + String OBS_API_METHOD_ATTR_KEY = "obs.apiMethod"; + String OBS_OWNER = "obs.owner"; + String OBS_LENGTH = "obs.length"; + String OBS_IS_LATEST = "obs.latest"; + String OBS_API_METHOD_PUT_OBJECT = "PutOBSObject"; + String OBS_API_METHOD_MULTIPART_UPLOAD = "obs.multipartUpload"; + String OBS_SSE_ALGORITHM = "obs.algorithm"; + String OBS_ENCRYPTION_STRATEGY = "obs.encryptionStrategy"; + String OBS_EXPIRATION_TIME = "obs.expirationTime"; + String OBS_ERROR_Message = "obs.errorMessage"; + String OBS_ERROR_CODE = "obs.errorCode"; + String OBS_STATUS_CODE = "obs.statusCode"; + String OBS_ADDITIONAL_DETAILS = "obs.additionalDetails"; + String OBS_EXCEPTION = "obs.exception"; + String OBS_PROCESS_UNSCHEDULED_MESSAGE = "Processor unscheduled, stopping upload"; + String CONTENT_DISPOSITION_ATTACHMENT = "attachment; filename="; + String HASH_ALGORITHM = "hash.algorithm"; + String HASH_VALUE = "hash.value"; + + ProxySpec[] PROXY_SPECS = {ProxySpec.HTTP_AUTH}; +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/DeleteOBSObject.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/DeleteOBSObject.java new file mode 100644 index 0000000000000000000000000000000000000000..3c305061fb43ada1b49d2cac7f624da90bf51038 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/DeleteOBSObject.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.obs; + +import com.obs.services.ObsClient; +import com.obs.services.exception.ObsException; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processors.huawei.abstractprocessor.AbstractOBSProcessor; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; + + +@SupportsBatching +@WritesAttributes({ + @WritesAttribute(attribute = Constants.OBS_EXCEPTION, description = "The class name of the exception thrown during processor execution"), + @WritesAttribute(attribute = Constants.OBS_ADDITIONAL_DETAILS, description = "The OBS supplied detail from the failed operation"), + @WritesAttribute(attribute = Constants.OBS_STATUS_CODE, description = "The HTTP error code (if available) from the failed operation"), + @WritesAttribute(attribute = Constants.OBS_ERROR_CODE, description = "The OBS moniker of the failed operation"), + @WritesAttribute(attribute = Constants.OBS_ERROR_Message, description = "The OBS exception message from the failed operation")}) +@Tags({"HuaweiCloud", "obs", "Archive", "Delete"}) +@InputRequirement(Requirement.INPUT_REQUIRED) +@CapabilityDescription("Deletes FlowFiles on an HuaweiCloud OBS Bucket. " + + "If attempting to delete a file that does not exist, FlowFile is routed to success.") +public class DeleteOBSObject extends AbstractOBSProcessor { + public static final List properties = Collections.unmodifiableList( + Arrays.asList(OBS_REGION, ENDPOINT_OVERRIDE_URL, BUCKET, ACCESS_KEY, SECRET_KEY, HUAWEI_CREDENTIALS_PROVIDER_SERVICE, KEY, TIMEOUT, +// SSL_CONTEXT_SERVICE, + PROXY_CONFIGURATION_SERVICE)); + + @Override + public List getSupportedPropertyDescriptors() { + return properties; + } + + @Override + public void onTrigger(final ProcessContext context, final ProcessSession session) { + FlowFile flowFile = session.get(); + if (flowFile == null) { + return; + } + + final long startNanos = System.nanoTime(); + + final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue(); + final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue(); + final String region = context.getProperty(OBS_REGION).getValue(); + final String endpoint = context.getProperty(ENDPOINT_OVERRIDE_URL).getValue(); + + final ObsClient obsClient = getClient(); + // Deletes a key + try { + obsClient.deleteObject(bucket, key); + } catch (final ObsException e) { + flowFile = extractExceptionDetails(e, session, flowFile); + getLogger().error("Failed to delete OBS Object for {}; routing to failure", flowFile, e); + flowFile = session.penalize(flowFile); + session.transfer(flowFile, REL_FAILURE); + return; + } + + session.transfer(flowFile, REL_SUCCESS); + final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); + getLogger().info("Successfully delete OBS Object for {} in {} millis; routing to success", flowFile, transferMillis); + session.getProvenanceReporter().invokeRemoteProcess(flowFile, OBSUtils.getUrl(region,endpoint, bucket, key), "Object deleted"); + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/FetchOBSObject.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/FetchOBSObject.java new file mode 100644 index 0000000000000000000000000000000000000000..d8a6fe5cef8eed6d4a79cb38e34a2352308fa39a --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/FetchOBSObject.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.obs; + +import com.obs.services.ObsClient; +import com.obs.services.model.GetObjectMetadataRequest; +import com.obs.services.model.GetObjectRequest; +import com.obs.services.model.ObjectMetadata; +import com.obs.services.model.ObsObject; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.ConfigVerificationResult.Outcome; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.DataUnit; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.huawei.abstractprocessor.AbstractOBSProcessor; + +import java.io.IOException; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; +import static org.apache.nifi.processors.huawei.obs.Constants.*; + +@SupportsBatching +@InputRequirement(Requirement.INPUT_REQUIRED) +@Tags({"HuaweiCloud", "OBS", "Get", "Fetch"}) +@CapabilityDescription("Retrieves the contents of an OBS Object and writes it to the content of a FlowFile") +@WritesAttributes({ + @WritesAttribute(attribute = OBS_BUCKET, description = "The name of the OBS bucket"), + @WritesAttribute(attribute = OBS_OPERATION, description = "The name of the operator"), + @WritesAttribute(attribute = "path", description = "The path of the file"), + @WritesAttribute(attribute = "absolute.path", description = "The path of the file"), + @WritesAttribute(attribute = "filename", description = "The name of the file"), + @WritesAttribute(attribute = "mime.type", description = "If OBS provides the content type/MIME type, this attribute will hold that file"), + @WritesAttribute(attribute = OBS_E_TAG, description = "The ETag that can be used to see if the file has changed"), + @WritesAttribute(attribute = OBS_EXCEPTION, description = "The class name of the exception thrown during processor execution"), + @WritesAttribute(attribute = OBS_ADDITIONAL_DETAILS, description = "The OBS supplied detail from the failed operation"), + @WritesAttribute(attribute = OBS_STATUS_CODE, description = "The HTTP error code (if available) from the failed operation"), + @WritesAttribute(attribute = OBS_ERROR_CODE, description = "The OBS moniker of the failed operation"), + @WritesAttribute(attribute = OBS_ERROR_Message, description = "The OBS exception message from the failed operation"), + @WritesAttribute(attribute = OBS_EXPIRATION_TIME, description = "If the file has an expiration date, this attribute will be set, containing the milliseconds since epoch in UTC time"), + @WritesAttribute(attribute = OBS_SSE_ALGORITHM, description = "The server side encryption algorithm of the object"), + @WritesAttribute(attribute = OBS_VERSION, description = "The version of the OBS object"), + @WritesAttribute(attribute = OBS_ENCRYPTION_STRATEGY, description = "The name of the encryption strategy that was used to store the OBS object (if it is encrypted)"),}) +public class FetchOBSObject extends AbstractOBSProcessor { + public static final PropertyDescriptor RANGE_START = new PropertyDescriptor.Builder() + .name("range-start") + .displayName("Range Start") + .description("The byte position at which to start reading from the object. An empty value or a value of " + + "zero will start reading at the beginning of the object.") + .addValidator(StandardValidators.DATA_SIZE_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .build(); + + public static final PropertyDescriptor RANGE_LENGTH = new PropertyDescriptor.Builder() + .name("range-length") + .displayName("Range Length") + .description("The number of bytes to download from the object, starting from the Range Start. An empty " + + "value or a value that extends beyond the end of the object will read to the end of the object.") + .addValidator(StandardValidators.createDataSizeBoundsValidator(1, Long.MAX_VALUE)) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .build(); + + public static final List properties = Collections.unmodifiableList( + Arrays.asList( + OBS_REGION, + ENDPOINT_OVERRIDE_URL, + BUCKET, + KEY, + ACCESS_KEY, + SECRET_KEY, + HUAWEI_CREDENTIALS_PROVIDER_SERVICE, + ENCRYPTION_SERVICE, + TIMEOUT, + RANGE_START, + RANGE_LENGTH, + PROXY_CONFIGURATION_SERVICE)); + + @Override + public List getSupportedPropertyDescriptors() { + return properties; + } + + @Override + public List verify(ProcessContext context, ComponentLog verificationLogger, Map attributes) { + final List results = new ArrayList<>(super.verify(context, verificationLogger, attributes)); + + final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(attributes).getValue(); + final String key = context.getProperty(KEY).evaluateAttributeExpressions(attributes).getValue(); + + final ObsClient client = OBSUtils.createClient(context, getLogger()); + final GetObjectMetadataRequest request = createGetObjectMetadataRequest(context, attributes); + + try { + final ObjectMetadata objectMetadata = client.getObjectMetadata(request); + final long byteCount = objectMetadata.getContentLength(); + results.add(new ConfigVerificationResult.Builder() + .verificationStepName("HEAD OBS Object") + .outcome(Outcome.SUCCESSFUL) + .explanation(String.format("Successfully performed HEAD on [%s] (%s bytes) from Bucket [%s]", key, byteCount, bucket)) + .build()); + } catch (final Exception e) { + getLogger().error(String.format("Failed to fetch [%s] from Bucket [%s]", key, bucket), e); + results.add(new ConfigVerificationResult.Builder() + .verificationStepName("HEAD OBS Object") + .outcome(Outcome.FAILED) + .explanation(String.format("Failed to perform HEAD on [%s] from Bucket [%s]: %s", key, bucket, e.getMessage())) + .build()); + } + + return results; + } + + @Override + public void onTrigger(final ProcessContext context, final ProcessSession session) { + getLogger().warn("FetchObs Trigger {}", System.currentTimeMillis()); + FlowFile flowFile = session.get(); + if (flowFile == null) { + return; + } + + final long startNanos = System.nanoTime(); + final Map attributes = new HashMap<>(); + + final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue(); + final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue(); + final String region = context.getProperty(OBS_REGION).getValue(); + final String endpoint = context.getProperty(ENDPOINT_OVERRIDE_URL).getValue(); + + final ObsClient client = getClient(); + final GetObjectRequest request = createGetObjectRequest(context, flowFile.getAttributes(), attributes); + try { + final ObsObject obsObject = client.getObject(request); + if (obsObject == null) { + throw new IOException("HuaweiCloud refused to execute this request."); + } + flowFile = session.importFrom(obsObject.getObjectContent(), flowFile); + attributes.put(OBS_OBJECT, key); + attributes.put(FILENAME, obsObject.getObjectKey()); + attributes.put(OBS_OPERATION, "FETCH"); + attributes.put(OBS_BUCKET, obsObject.getBucketName()); + final ObjectMetadata metadata = obsObject.getMetadata(); + if (StringUtils.isNotBlank(metadata.getContentDisposition())) { + final String contentDisposition = URLDecoder.decode(metadata.getContentDisposition(), StandardCharsets.UTF_8.name()); + if (contentDisposition.equals(PutOBSObject.CONTENT_DISPOSITION_INLINE) || contentDisposition.startsWith(CONTENT_DISPOSITION_ATTACHMENT)) { + setFilePathAttributes(attributes, key); + } else { + setFilePathAttributes(attributes, contentDisposition); + } + } + if (metadata.getContentMd5() != null) { + attributes.put(HASH_VALUE, metadata.getContentMd5()); + attributes.put(HASH_ALGORITHM, "MD5"); + } + if (metadata.getContentType() != null) { + attributes.put(CoreAttributes.MIME_TYPE.key(), metadata.getContentType()); + } + if (metadata.getEtag() != null) { + attributes.put(OBS_E_TAG, metadata.getEtag()); + } + if (metadata.getExpires() != null) { + attributes.put(OBS_EXPIRATION_TIME, String.valueOf(metadata.getExpires())); + } + if (metadata.getAllMetadata() != null) { + attributes.putAll(metadata.getAllMetadata()); + } + } catch (Exception ioe) { + flowFile = extractExceptionDetails(ioe, session, flowFile); + getLogger().error("Failed to retrieve OBS Object for {}; routing to failure", flowFile, ioe); + flowFile = session.penalize(flowFile); + session.transfer(flowFile, REL_FAILURE); + return; + } + if (!attributes.isEmpty()) { + flowFile = session.putAllAttributes(flowFile, attributes.entrySet().stream(). + collect(Collectors.toMap(item -> String.valueOf(item.getKey()) , + val -> String.valueOf(val.getValue())))); + } + + session.transfer(flowFile, REL_SUCCESS); + final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); + getLogger().info("Successfully retrieved OBS Object for {} in {} millis; routing to success", new Object[]{flowFile, transferMillis}); + session.getProvenanceReporter().fetch(flowFile, OBSUtils.getUrl(region, endpoint, bucket, key), transferMillis); + } + + private GetObjectMetadataRequest createGetObjectMetadataRequest(final ProcessContext context, final Map attributes) { + final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(attributes).getValue(); + final String key = context.getProperty(KEY).evaluateAttributeExpressions(attributes).getValue(); + return new GetObjectMetadataRequest(bucket, key); + } + + private GetObjectRequest createGetObjectRequest(final ProcessContext context, final Map flowFileAttributes, Map attributes) { + final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFileAttributes).getValue(); + final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFileAttributes).getValue(); + final long rangeStart = (context.getProperty(RANGE_START).isSet() ? context.getProperty(RANGE_START).evaluateAttributeExpressions(flowFileAttributes).asDataSize(DataUnit.B).longValue() : 0L); + final Long rangeLength = (context.getProperty(RANGE_LENGTH).isSet() ? context.getProperty(RANGE_LENGTH).evaluateAttributeExpressions(flowFileAttributes).asDataSize(DataUnit.B).longValue() : null); + final GetObjectRequest request = new GetObjectRequest(bucket, key); + + // Since the effect of the byte range 0- is equivalent to not sending a + // byte range and works for both zero and non-zero length objects, + // the single argument setRange() only needs to be called when the + // first byte position is greater than zero. + if (rangeLength != null) { + request.setRangeStart(rangeStart); + request.setRangeEnd(rangeStart + rangeLength - 1); + } else if (rangeStart > 0) { + request.setRangeStart(rangeStart); + } + + final ObsServiceEncryptionService encryptionService = context.getProperty(ENCRYPTION_SERVICE).asControllerService(ObsServiceEncryptionService.class); + if (encryptionService != null) { + encryptionService.configureGetObjectRequest(request, new ObjectMetadata()); + } + + if (request.getSseCHeader() !=null && request.getSseCHeader().getSSEAlgorithm() != null) { + String sseAlgorithmName = request.getSseCHeader().getSSEAlgorithm().name(); + attributes.put(OBS_SSE_ALGORITHM, sseAlgorithmName); + attributes.put(OBS_ENCRYPTION_STRATEGY, ObsServiceEncryptionService.STRATEGY_NAME_SSE_C); + } + return request; + } + + protected void setFilePathAttributes(Map attributes, String filePathName) { + final int lastSlash = filePathName.lastIndexOf("/"); + if (lastSlash > -1 && lastSlash < filePathName.length() - 1) { + attributes.put(CoreAttributes.PATH.key(), filePathName.substring(0, lastSlash)); + attributes.put(CoreAttributes.ABSOLUTE_PATH.key(), filePathName); + attributes.put(CoreAttributes.FILENAME.key(), filePathName.substring(lastSlash + 1)); + } else { + attributes.put(CoreAttributes.FILENAME.key(), filePathName); + } + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/ListOBSObject.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/ListOBSObject.java new file mode 100644 index 0000000000000000000000000000000000000000..3bf50e8e25aa3561c02154ca504e092c3a94eddf --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/ListOBSObject.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.obs; + +import com.obs.services.ObsClient; +import com.obs.services.model.GetObjectMetadataRequest; +import com.obs.services.model.ListVersionsResult; +import com.obs.services.model.ObjectMetadata; +import com.obs.services.model.VersionOrDeleteMarker; +import org.apache.nifi.annotation.behavior.*; +import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.state.Scope; +import org.apache.nifi.context.PropertyContext; +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.processor.DataUnit; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.util.list.ListableEntity; +import org.apache.nifi.processor.util.list.ListedEntityTracker; +import org.apache.nifi.processors.huawei.obs.model.OBSObjectBucketLister; +import org.apache.nifi.processors.huawei.obs.model.OBSRecord; +import org.apache.nifi.processors.huawei.obs.model.ObsBucketLister; +import org.apache.nifi.serialization.record.RecordSchema; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.TimeUnit; + +import static org.apache.nifi.processors.huawei.obs.Constants.*; +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; + +@PrimaryNodeOnly +@TriggerSerially +@TriggerWhenEmpty +@InputRequirement(Requirement.INPUT_FORBIDDEN) +@Tags({"Huawei", "OBS", "list"}) +@CapabilityDescription("Retrieves a listing of objects from an obs bucket. For each object that is listed, creates a FlowFile that represents " + + "the object so that it can be fetched in conjunction with FetchOBSObject. This Processor is designed to run on Primary Node only " + + "in a cluster. If the primary node changes, the new Primary Node will pick up where the previous node left off without duplicating " + + "all of the data.") +@Stateful(scopes = Scope.CLUSTER, description = "After performing a listing of keys, the timestamp of the newest key is stored, " + + "along with the keys that share that same timestamp. This allows the Processor to list only keys that have been added or modified after " + + "this date the next time that the Processor is run. State is stored across the cluster so that this Processor can be run on Primary Node only and if a new Primary " + + "Node is selected, the new node can pick up where the previous node left off, without duplicating the data.") +@WritesAttributes({ + @WritesAttribute(attribute = OBS_BUCKET, description = "The name of the OBS bucket"), + @WritesAttribute(attribute = "filename", description = "The name of the file"), + @WritesAttribute(attribute = OBS_E_TAG, description = "The ETag that can be used to see if the file has changed"), + @WritesAttribute(attribute = OBS_OWNER, description = "Object owner"), + @WritesAttribute(attribute = OBS_IS_LATEST, description = "A boolean indicating if this is the latest version of the object"), + @WritesAttribute(attribute = OBS_LAST_MODIFIED, description = "The last modified time in milliseconds since epoch in UTC time"), + @WritesAttribute(attribute = OBS_LENGTH, description = "The size of the object in bytes"), + @WritesAttribute(attribute = OBS_STORAGE_CLASS, description = "The storage class of the object"), + @WritesAttribute(attribute = OBS_VERSION, description = "The version of the object, if applicable"), + @WritesAttribute(attribute = OBS_USER_META + ".__", description = "If 'Write User Metadata' is set to 'True', the user defined metadata associated to the OBS object that is being listed " + + "will be written as part of the flowfile attributes")}) +public class ListOBSObject extends AbstractListHuaweiProcessor { + public static final List properties = Collections.unmodifiableList(Arrays.asList( + OBS_REGION, + ENDPOINT_OVERRIDE_URL, + BUCKET, + ACCESS_KEY, + SECRET_KEY, + HUAWEI_CREDENTIALS_PROVIDER_SERVICE, + DELIMITER, + PREFIX, + LISTING_STRATEGY, + ListedEntityTracker.TRACKING_STATE_CACHE, + ListedEntityTracker.TRACKING_TIME_WINDOW, + ListedEntityTracker.INITIAL_LISTING_TARGET, + RECORD_WRITER, + MAX_AGE, + MIN_AGE, + MAX_SIZE, + MIN_SIZE, + WRITE_USER_METADATA, + TIMEOUT, + PROXY_CONFIGURATION_SERVICE + )); + + public static final Set relationships = Collections.singleton(REL_SUCCESS); + + protected volatile ObsClient client; + + @OnScheduled + public void onScheduled(final ProcessContext context) { + client = OBSUtils.createClient(context, getLogger()); + } + + @Override + protected Map createAttributes(ListableEntity entity, ProcessContext context) { + OBSRecord record = (OBSRecord) entity; + VersionOrDeleteMarker versionSummary = record.getVersionSummary(); + if (versionSummary == null) { + return new HashMap<>(); + } + final Map attributes = new HashMap<>(); + attributes.put(CoreAttributes.FILENAME.key(), versionSummary.getObjectKey()); + attributes.put(OBS_BUCKET, versionSummary.getBucketName()); + if (versionSummary.getOwner() != null) { + attributes.put(OBS_OWNER, versionSummary.getOwner().getId()); + } + attributes.put(OBS_E_TAG, versionSummary.getEtag()); + attributes.put(OBS_LAST_MODIFIED, String.valueOf(versionSummary.getLastModified().getTime())); + attributes.put(OBS_LENGTH, String.valueOf(versionSummary.getSize())); + attributes.put(OBS_STORAGE_CLASS, versionSummary.getStorageClass()); + attributes.put(OBS_IS_LATEST, String.valueOf(versionSummary.isLatest())); + if (versionSummary.getVersionId() != null) { + attributes.put(OBS_VERSION, versionSummary.getVersionId()); + } + // Add user-defined metadata + if (record.getObjectMetadata() != null) { + for (Map.Entry e : record.getObjectMetadata().getAllMetadata().entrySet()) { + attributes.put(OBS_USER_META + "." + e.getKey(), (String) e.getValue()); + } + } + return attributes; + } + + @Override + protected String getPath(ProcessContext context) { + return context.getProperty(BUCKET).evaluateAttributeExpressions().getValue(); + } + + @Override + protected List performListing(ProcessContext context, Long minTimestamp, ListingMode listingMode) throws IOException { + final long minimumTimestamp = minTimestamp == null ? 0 : minTimestamp; + final String region = context.getProperty(OBS_REGION).getValue(); + final String endpoint = context.getProperty(ENDPOINT_OVERRIDE_URL).getValue(); + List list = new ArrayList<>(); + int batchCount = 0; + ObsBucketLister bucketLister = new OBSObjectBucketLister(client, context); + do { + ListVersionsResult versionListing = bucketLister.listVersions(); + getLogger().warn("ListVersionsResult count : {}", versionListing.getVersions().length); + for (VersionOrDeleteMarker versionSummary : versionListing.getVersions()) { + if (!isFileInfoMatchesWithAgeAndSize(context, minimumTimestamp, versionSummary.getLastModified().getTime(), versionSummary.getSize())) { + continue; + } + ObjectMetadata objectMetadata = getObjectMetadata(context, client, versionSummary); + OBSRecord record = new OBSRecord(); + record.setObjectMetadata(objectMetadata); + record.setRegion(region); + record.setEndpoint(endpoint); + record.setVersionSummary(versionSummary); + list.add(record); + batchCount++; + } + bucketLister.setNextMarker(); + getLogger().info("Successfully listed {} new files from OBS; routing to success", batchCount); + batchCount = 0; + // 是否完成 + } while (bucketLister.isTruncated()); + return list; + } + + protected boolean isFileInfoMatchesWithAgeAndSize(final ProcessContext context, final long minimumTimestamp, + final long lastModified, final long size) { + final long minSize = context.getProperty(MIN_SIZE).asDataSize(DataUnit.B).longValue(); + final Double maxSize = context.getProperty(MAX_SIZE).asDataSize(DataUnit.B); + final long minAge = context.getProperty(MIN_AGE).asTimePeriod(TimeUnit.MILLISECONDS); + final Long maxAge = context.getProperty(MAX_AGE).asTimePeriod(TimeUnit.MILLISECONDS); + + if (lastModified < minimumTimestamp) { + return false; + } + final long fileAge = System.currentTimeMillis() - lastModified; + if (minAge > fileAge) { + return false; + } + if (maxAge != null && maxAge < fileAge) { + return false; + } + if (minSize > size) { + return false; + } + return maxSize == null || maxSize >= size; + } + + @Override + protected boolean isListingResetNecessary(PropertyDescriptor property) { + return BUCKET.equals(property) + || OBS_REGION.equals(property); + } + + @Override + protected Scope getStateScope(PropertyContext context) { + return null; + } + + @Override + protected RecordSchema getRecordSchema() { + return null; + } + + @Override + protected Integer countUnfilteredListing(ProcessContext context) throws IOException { + return null; + } + + @Override + protected String getListingContainerName(ProcessContext context) { + return null; + } + + @Override + protected List getSupportedPropertyDescriptors() { + return properties; + } + + @Override + public Set getRelationships() { + return relationships; + } + + private ObjectMetadata getObjectMetadata(ProcessContext context, ObsClient client, VersionOrDeleteMarker versionSummary) { + ObjectMetadata objectMetadata = null; + if (context.getProperty(WRITE_USER_METADATA).asBoolean()) { + try { + GetObjectMetadataRequest request = new GetObjectMetadataRequest(versionSummary.getBucketName(), versionSummary.getKey()); + objectMetadata = client.getObjectMetadata(request); + } catch (final Exception e) { + getLogger().warn("Failed to obtain User Metadata for Obs Object {} in bucket {}. Will list Obs Object without the user metadata", + new Object[]{versionSummary.getKey(), versionSummary.getBucketName()}, e); + } + } + return objectMetadata; + } +} + diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/OBSRegions.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/OBSRegions.java new file mode 100644 index 0000000000000000000000000000000000000000..211ad06b81da2a700180b25be2a60d841949546d --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/OBSRegions.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.huawei.obs; + +import org.apache.nifi.components.AllowableValue; + +import java.util.ArrayList; +import java.util.List; + +public enum OBSRegions { + NOT_FOUND("NotFound", "NotFound", "Not found"), + OF_SOUTH_1("af-south-1", "obs.af-south-1.myhuaweicloud.com", "Africa - Johannesburg"), + CN_NORTH_4("cn-north-4", "obs.cn-north-4.myhuaweicloud.com", "North China - Beijing IV"), + CN_NORTH_1("cn-north-1", "obs.cn-north-1.myhuaweicloud.com", "North China - Beijing I"), + CN_NORTH_9("cn-north-9", "obs.cn-north-9.myhuaweicloud.com", "North China - Ulanqabu I"), + CN_EAST_2("cn-east-2", "obs.cn-east-2.myhuaweicloud.com", "East China - Shanghai II"), + CN_EAST_3("cn-east-3", "obs.cn-east-3.myhuaweicloud.com", "East China - Shanghai I"), + CN_SOUTH_1("cn-south-1", "obs.cn-south-1.myhuaweicloud.com", "South China - Guangzhou"), + LA_NORTH_2("la-north-2", "obs.la-north-2.myhuaweicloud.com", "Latin America - Mexico City II"), + NA_MEXICO_1("na-mexico-1", "obs.na-mexico-1.myhuaweicloud.com", "Latin America - Mexico City I"), + SA_BRAZIL_1("sa-brazil-1", "obs.sa-brazil-1.myhuaweicloud.com", "Latin America - Sao Paulo I"), + CN_SOUTHWEST_2("cn-southwest-2", "obs.cn-southwest-2.myhuaweicloud.com", "Southwest - Guiyang I"), + AP_SOUTHWEST_2("ap-southeast-2", "obs.ap-southeast-2.myhuaweicloud.com", "Asia Pacific - Bangkok"), + AP_SOUTHWEST_3("ap-southeast-3", "obs.ap-southeast-3.myhuaweicloud.com", "Asia Pacific - Singapore"), + AP_SOUTHWEST_1("ap-southeast-1", "obs.ap-southeast-1.myhuaweicloud.com", "China - Hong Kong"); + + public static final OBSRegions DEFAULT_REGION = CN_NORTH_1; + private final String name; + + private final String endpoint; + + private final String description; + + OBSRegions(String name, String endpoint, String description) { + this.name = name; + this.endpoint = endpoint; + this.description = description; + } + + public String getName() { + return this.name; + } + + public String getEndpoint() { + return this.endpoint; + } + + public String getDescription() { + return this.description; + } + + public static OBSRegions fromName(String regionName) { + OBSRegions[] var1 = values(); + int var2 = var1.length; + + for (int var3 = 0; var3 < var2; ++var3) { + OBSRegions region = var1[var3]; + if (region.getName().equals(regionName)) { + return region; + } + } + + throw new IllegalArgumentException("Cannot create enum from " + regionName + " value!"); + } + + public static AllowableValue[] getAvailableOBSRegions() { + final List values = new ArrayList<>(); + for (final OBSRegions region : OBSRegions.values()) { + AllowableValue allowableValue = new AllowableValue(region.getName(), region.getDescription(), "Huawei Obs Region Code : " + region.getName()); + values.add(allowableValue); + } + return values.toArray(new AllowableValue[0]); + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/OBSUtils.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/OBSUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..e63a2bc732963b1918bf6566ee3f2e3cac11dab5 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/OBSUtils.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.huawei.obs; + +import com.obs.services.ObsClient; +import com.obs.services.ObsConfiguration; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processors.huawei.credentials.provider.service.HuaweiCredentialsProviderService; +import org.apache.nifi.proxy.ProxyConfiguration; +import org.apache.nifi.proxy.ProxyConfigurationService; +import org.apache.nifi.ssl.SSLContextService; + +import javax.net.ssl.SSLContext; +import java.net.Proxy; +import java.util.concurrent.TimeUnit; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; + +public class OBSUtils { + public static String getUrl(String region,String endpoint, String bucket, String objectKey) { + if (StringUtils.isBlank(region) || StringUtils.isBlank(bucket) || StringUtils.isBlank(objectKey)) { + return null; + } + + if (objectKey.startsWith("/")) { + objectKey = objectKey.substring(1); + } + + if (!OBSRegions.NOT_FOUND.equals(OBSRegions.fromName(region))) { + endpoint = OBSRegions.fromName(region).getEndpoint(); + } + return String.format("http://%s.%s/%s", bucket, endpoint, objectKey); + } + + public static ObsClient createObsClientWithAkSk(ProcessContext context, ComponentLog logger) { + final String accessKey = context.getProperty(ACCESS_KEY).evaluateAttributeExpressions().getValue(); + final String secretKey = context.getProperty(SECRET_KEY).evaluateAttributeExpressions().getValue(); + ObsClient obsClient = new ObsClient(accessKey, secretKey, createConfiguration(context)); + if (logger != null) { + logger.warn("create ObsClient success"); + } + return obsClient; + } + + /** + * Attempts to create the client using the controller service first before falling back to the standard configuration. + * + * @param context The process context + * @return The created client + */ + public static ObsClient createClient(final ProcessContext context, ComponentLog logger) { + final ControllerService service = context.getProperty(HUAWEI_CREDENTIALS_PROVIDER_SERVICE).asControllerService(); + if (service != null) { + logger.debug("Using Huawei credentials provider service for creating client"); + final HuaweiCredentialsProviderService huaweiCredentialsProviderService = + context.getProperty(HUAWEI_CREDENTIALS_PROVIDER_SERVICE).asControllerService(HuaweiCredentialsProviderService.class); + return new ObsClient(huaweiCredentialsProviderService.getCredentialsProvider(), createConfiguration(context)); + } else { + logger.debug("Using Huawei credentials for creating client"); + return createObsClientWithAkSk(context, logger); + } + } + + private static ObsConfiguration createConfiguration(final ProcessContext context) { + final ObsConfiguration config = new ObsConfiguration(); + + config.setMaxConnections(context.getMaxConcurrentTasks()); + config.setMaxErrorRetry(0); + // If this is changed to be a property, ensure other uses are also changed + final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue(); + config.setConnectionTimeout(commsTimeout); + config.setSocketTimeout(commsTimeout); + // not set + if(context.getProperty(SSL_CONTEXT_SERVICE).isSet()) { + final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); + if (sslContextService != null) { + final SSLContext sslContext = sslContextService.createContext(); + config.setSslProvider(sslContext.getProvider().getName()); + } + } + + final ProxyConfiguration proxyConfig = ProxyConfiguration.getConfiguration(context, () -> { + if (context.getProperty(ProxyConfigurationService.PROXY_CONFIGURATION_SERVICE).isSet()) { + final ProxyConfigurationService configurationService = context.getProperty(ProxyConfigurationService.PROXY_CONFIGURATION_SERVICE).asControllerService(ProxyConfigurationService.class); + return configurationService.getConfiguration(); + } + return ProxyConfiguration.DIRECT_CONFIGURATION; + }); + + if (Proxy.Type.HTTP.equals(proxyConfig.getProxyType())) { + config.setHttpProxy(proxyConfig.getProxyServerHost(), proxyConfig.getProxyServerPort(), proxyConfig.getProxyUserName(), proxyConfig.getProxyUserPassword()); + } + + String name = context.getProperty(OBS_REGION).getValue(); + OBSRegions region = OBSRegions.fromName(name); + String endpoint = context.getProperty(ENDPOINT_OVERRIDE_URL).getValue(); + if (!OBSRegions.NOT_FOUND.equals(region)) { + endpoint = region.getEndpoint(); + } + config.setEndPoint(endpoint); + return config; + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/PutOBSObject.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/PutOBSObject.java new file mode 100644 index 0000000000000000000000000000000000000000..475289322894efe444d979f0359feef1a7ab50d4 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/PutOBSObject.java @@ -0,0 +1,1158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.obs; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.obs.services.ObsClient; +import com.obs.services.exception.ObsException; +import com.obs.services.model.*; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.annotation.behavior.*; +import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.processor.DataUnit; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.io.InputStreamCallback; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.huawei.abstractprocessor.AbstractOBSProcessor; +import org.apache.nifi.processors.huawei.obs.encryption.StandardOBSEncryptionService; + +import java.io.*; +import java.net.URLEncoder; +import java.nio.file.Files; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; +import static org.apache.nifi.processors.huawei.obs.Constants.*; + +@SupportsBatching +@InputRequirement(Requirement.INPUT_REQUIRED) +@Tags({"HuaweiCloud", "OBS", "Archive", "Put"}) +@CapabilityDescription("Puts FlowFiles to an OBS Bucket.\n" + + "The upload uses either the PutOBSObject method or the PutOBSMultipartUpload method. The PutOBSObject method " + + "sends the file in a single synchronous call, but it has a 5GB size limit. Larger files are sent using the " + + "PutOBSMultipartUpload method. This multipart process " + + "saves state after each step so that a large upload can be resumed with minimal loss if the processor or " + + "cluster is stopped and restarted.\n" + + "A multipart upload consists of three steps:\n" + + " 1) initiate upload,\n" + + " 2) upload the parts, and\n" + + " 3) complete the upload.\n" + + "For multipart uploads, the processor saves state locally tracking the upload ID and parts uploaded, which " + + "must both be provided to complete the upload.\n" + + "The HuaweiCloud libraries select an endpoint URL based on the HuaweiCloud region, but this can be overridden with the " + + "'Endpoint Override URL' property for use with other OBS-compatible endpoints.\n" + + "The OBS API specifies that the maximum file size for a PutOBSObject upload is 5GB. It also requires that " + + "parts in a multipart upload must be at least 5MB in size, except for the last part. These limits " + + "establish the bounds for the Multipart Upload Threshold and Part Size properties.") +@DynamicProperty(name = "The name of a User-Defined Metadata field to add to the OBS Object", + value = "The value of a User-Defined Metadata field to add to the OBS Object", + description = "Allows user-defined metadata to be added to the OBS object as key/value pairs", + expressionLanguageScope = ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) +@ReadsAttribute(attribute = "filename", description = "Uses the FlowFile's filename as the filename for the OBS object") +@WritesAttributes({ + @WritesAttribute(attribute = OBS_BUCKET, description = "The OBS bucket where the Object was put in OBS"), + @WritesAttribute(attribute = OBS_OBJECT, description = "The OBS key within where the Object was put in OBS"), + @WritesAttribute(attribute = OBS_CONTENT_TYPE, description = "The OBS content type of the OBS Object that put in OBS"), + @WritesAttribute(attribute = OBS_VERSION, description = "The version of the OBS Object that was put to OBS"), + @WritesAttribute(attribute = OBS_EXCEPTION, description = "The class name of the exception thrown during processor execution"), + @WritesAttribute(attribute = OBS_ADDITIONAL_DETAILS, description = "The OBS supplied detail from the failed operation"), + @WritesAttribute(attribute = OBS_STATUS_CODE, description = "The HTTP error code (if available) from the failed operation"), + @WritesAttribute(attribute = OBS_ERROR_CODE, description = "The OBS moniker of the failed operation"), + @WritesAttribute(attribute = OBS_ERROR_Message, description = "The OBS exception message from the failed operation"), + @WritesAttribute(attribute = OBS_E_TAG, description = "The ETag of the OBS Object"), + @WritesAttribute(attribute = OBS_CONTENT_DISPOSITION, description = "The content disposition of the OBS Object that put in OBS"), + @WritesAttribute(attribute = OBS_CACHE_CONTROL, description = "The cache-control header of the OBS Object"), + @WritesAttribute(attribute = OBS_UPLOAD_ID, description = "The uploadId used to upload the Object to OBS"), + @WritesAttribute(attribute = OBS_EXPIRATION_TIME, description = "A human-readable form of the expiration date of " + + "the OBS object, if one is set"), + @WritesAttribute(attribute = OBS_SSE_ALGORITHM, description = "The server side encryption algorithm of the object"), + @WritesAttribute(attribute = OBS_USER_META, description = "A human-readable form of the User Metadata of " + + "the OBS object, if any was set"), + @WritesAttribute(attribute = OBS_ENCRYPTION_STRATEGY, description = "The name of the encryption strategy, if any was set"),}) +public class PutOBSObject extends AbstractOBSProcessor { + public static final long MIN_PART_SIZE = 50L * 1024L * 1024L; + public static final long MAX_PUTOBJECT_SIZE = 5L * 1024L * 1024L * 1024L; + public static final String CONTENT_DISPOSITION_INLINE = "inline"; + public static final String CONTENT_DISPOSITION_ATTACHMENT = "attachment"; + + public static final String CONTENT_DISPOSITION_FILENAME = "filename"; + + private static final Set STORAGE_CLASSES = Collections.unmodifiableSortedSet(new TreeSet<>( + Arrays.stream(StorageClassEnum.values()).map(StorageClassEnum::getCode).collect(Collectors.toSet()) + )); + + public static final PropertyDescriptor CONTENT_TYPE = new PropertyDescriptor.Builder() + .name("Content Type") + .displayName("Content Type") + .description("Sets the Content-Type HTTP header indicating the type of content stored in the associated " + + "object. The value of this header is a standard MIME type.\n" + + "OBS Java client will attempt to determine the correct content type if one hasn't been set" + + " yet. Users are responsible for ensuring a suitable content type is set when uploading streams. If " + + "no content type is provided and cannot be determined by the filename, the default content type " + + "\"application/octet-stream\" will be used.") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor CONTENT_DISPOSITION = new PropertyDescriptor.Builder() + .name("Content Disposition") + .displayName("Content Disposition") + .description("Sets the Content-Disposition HTTP header indicating if the content is intended to be displayed inline or should be downloaded.\n " + + "Possible values are 'inline' or 'attachment' or 'filename'. If this property is not specified, object's content-disposition will be set to null. " + + "when filename is selected, object's content-disposition will be set to filename" + + "When 'attachment' is selected, '; filename=' plus object key are automatically appended to form final value 'attachment; filename=\"filename.jpg\"'.") + .required(false) + .allowableValues(CONTENT_DISPOSITION_INLINE, CONTENT_DISPOSITION_ATTACHMENT, CONTENT_DISPOSITION_FILENAME) + .build(); + + public static final PropertyDescriptor CACHE_CONTROL = new PropertyDescriptor.Builder() + .name("Cache Control") + .displayName("Cache Control") + .description("Sets the Cache-Control HTTP header indicating the caching directives of the associated object. Multiple directives are comma-separated.") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor STORAGE_CLASS = new PropertyDescriptor.Builder() + .name("Storage Class") + .required(true) + .allowableValues(STORAGE_CLASSES) + .defaultValue(StorageClassEnum.STANDARD.getCode()) + .build(); + + public static final PropertyDescriptor MULTIPART_THRESHOLD = new PropertyDescriptor.Builder() + .name("Multipart Threshold") + .description("Specifies the file size threshold for switch from the PutOBSObject API to the " + + "PutOBSMultipartUpload API. Flow files bigger than this limit will be sent using the stateful " + + "multipart process.\n" + + "The valid range is 50MB to 5GB.") + .required(true) + .defaultValue("5 GB") + .addValidator(StandardValidators.createDataSizeBoundsValidator(MIN_PART_SIZE, MAX_PUTOBJECT_SIZE)) + .build(); + + public static final PropertyDescriptor MULTIPART_PART_SIZE = new PropertyDescriptor.Builder() + .name("Multipart Part Size") + .description("Specifies the part size for use when the PutOBSMultipart Upload API is used.\n" + + "Flow files will be broken into chunks of this size for the upload process, but the last part " + + "sent can be smaller since it is not padded.\n" + + "The valid range is 50MB to 5GB.") + .required(true) + .defaultValue("5 GB") + .addValidator(StandardValidators.createDataSizeBoundsValidator(MIN_PART_SIZE, MAX_PUTOBJECT_SIZE)) + .build(); + + public static final PropertyDescriptor MULTIPART_AGEOFF_INTERVAL = new PropertyDescriptor.Builder() + .name("Multipart Upload AgeOff Interval") + .description("Specifies the interval at which existing multipart uploads in HuaweiCloud OBS will be evaluated " + + "for ageoff. When processor is triggered it will initiate the ageoff evaluation if this interval has been " + + "exceeded.") + .required(true) + .defaultValue("60 min") + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .build(); + + public static final PropertyDescriptor MULTIPART_MAX_AGE = new PropertyDescriptor.Builder() + .name("Multipart Upload Max Age Threshold") + .description("Specifies the maximum age for existing multipart uploads in HuaweiCloud OBS. When the ageoff " + + "process occurs, any upload older than this threshold will be aborted.") + .required(true) + .defaultValue("7 days") + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .build(); + + public static final PropertyDescriptor MULTIPART_TEMP_DIR = new PropertyDescriptor.Builder() + .name("obs-temporary-directory-multipart") + .displayName("Temporary Directory Multipart State") + .description("Directory in which, for multipart uploads, the processor will locally save the state tracking the upload ID and parts " + + "uploaded which must both be provided to complete the upload.") + .required(true) + .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR) + .defaultValue("${java.io.tmpdir}") + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .build(); + + public static final List properties = Collections.unmodifiableList( + Arrays.asList( + OBS_REGION, + ENDPOINT_OVERRIDE_URL, + BUCKET, + ACCESS_KEY, + SECRET_KEY, + HUAWEI_CREDENTIALS_PROVIDER_SERVICE, + KEY, + STORAGE_CLASS, + CONTENT_TYPE, + CONTENT_DISPOSITION, + CACHE_CONTROL, + TIMEOUT, + OWNER, + READ_USER_LIST, + FULL_CONTROL_USER_LIST, + READ_ACL_LIST, + WRITE_ACL_LIST, + CANNED_ACL, + MULTIPART_THRESHOLD, + MULTIPART_PART_SIZE, + MULTIPART_AGEOFF_INTERVAL, + MULTIPART_MAX_AGE, + MULTIPART_TEMP_DIR, + ENCRYPTION_SERVICE, + PROXY_CONFIGURATION_SERVICE + )); + + private volatile String tempDirMultipart = System.getProperty("java.io.tmpdir"); + + @OnScheduled + public void setTempDir(final ProcessContext context) { + this.tempDirMultipart = context.getProperty(MULTIPART_TEMP_DIR).evaluateAttributeExpressions().getValue(); + } + + @Override + public List getSupportedPropertyDescriptors() { + return properties; + } + + @Override + public PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { + return new PropertyDescriptor.Builder() + .name(propertyDescriptorName) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .dynamic(true) + .build(); + } + + @Override + public void onTrigger(final ProcessContext context, final ProcessSession session) { + FlowFile flowFile = session.get(); + if (flowFile == null) { + return; + } + try { + final long startNanos = System.nanoTime(); + CustomProperties customProperties = new CustomProperties(); + customProperties.setBucket(context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue()); + customProperties.setObjectKey(context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue()); + customProperties.setCacheKey(getIdentifier() + "/" + customProperties.getBucket() + "/" + customProperties.getObjectKey()); + customProperties.setFileName(flowFile.getAttributes().get(CoreAttributes.FILENAME.key())); + customProperties.setMultipartThreshold(context.getProperty(MULTIPART_THRESHOLD).asDataSize(DataUnit.B).longValue()); + customProperties.setMultipartPartSize(context.getProperty(MULTIPART_PART_SIZE).asDataSize(DataUnit.B).longValue()); + customProperties.setContentDisposition(context.getProperty(CONTENT_DISPOSITION).getValue()); + customProperties.setContentType(context.getProperty(CONTENT_TYPE).evaluateAttributeExpressions(flowFile).getValue()); + customProperties.setCacheControl(context.getProperty(CACHE_CONTROL).evaluateAttributeExpressions(flowFile).getValue()); + customProperties.setEncryptionService(context.getProperty(ENCRYPTION_SERVICE).asControllerService(ObsServiceEncryptionService.class)); + if (customProperties.getEncryptionService() == null) { + customProperties.setEncryptionService(new StandardOBSEncryptionService()); + } + customProperties.setStorageClass(StorageClassEnum.valueOf(context.getProperty(STORAGE_CLASS).getValue())); + customProperties.setFlowFile(flowFile); + + final Map attributes = new HashMap<>(); + attributes.put(OBS_BUCKET, customProperties.getBucket()); + attributes.put(OBS_OBJECT, customProperties.getObjectKey()); + + final long now = System.currentTimeMillis(); + + ageOffUploads(context, client, now, customProperties.getBucket()); + session.read(customProperties.getFlowFile(), new InputStreamCallback() { + @Override + public void process(final InputStream rawIn) throws IOException { + attributes.putAll(doUpload(customProperties, rawIn, client, context)); + } + }); + + if (!attributes.isEmpty()) { + flowFile = session.putAllAttributes(flowFile, attributes); + } + session.transfer(flowFile, REL_SUCCESS); + final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); + session.getProvenanceReporter().send(flowFile, attributes.get(OBJECT_URL), millis); + getLogger().info("Successfully put {} to OBS in {} milliseconds", customProperties.getFlowFile(), millis); + try { + removeLocalState(customProperties.getCacheKey()); + } catch (IOException e) { + getLogger().info("Error trying to delete key {} from cache: {}", + customProperties.getCacheKey(), e.getMessage()); + } + } catch (final ProcessException | ObsException pe) { + extractExceptionDetails(pe, session, flowFile); + getLogger().error("Failed to put {} to Huawei OBS due to {}", new Object[]{flowFile, pe}); + flowFile = session.penalize(flowFile); + session.transfer(flowFile, REL_FAILURE); + } + } + + public File getPersistenceFile() { + return new File(this.tempDirMultipart + File.separator + getIdentifier()); + } + + protected boolean localUploadExistsInOBS(final ObsClient obsClient, final String bucket, final MultipartState localState) { + ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest(bucket); + MultipartUploadListing listing = obsClient.listMultipartUploads(listRequest); + for (MultipartUpload upload : listing.getMultipartTaskList()) { + if (upload.getUploadId().equals(localState.getUploadId())) { + return true; + } + } + return false; + } + + public synchronized MultipartState getLocalStateIfInOBS(final ObsClient obsClient, final String bucket, + final String objectKey) throws IOException { + MultipartState currState = getLocalState(objectKey); + if (currState == null) { + return null; + } + + if (localUploadExistsInOBS(obsClient, bucket, currState)) { + getLogger().info("Local state for {} loaded with uploadId {} and {} partETags", + objectKey, currState.getUploadId(), currState.getPartETags().size()); + return currState; + } else { + getLogger().info("Local state for {} with uploadId {} does not exist in OBS, deleting local state", + objectKey, currState.getUploadId()); + persistLocalState(objectKey, null); + return null; + } + } + + protected synchronized MultipartState getLocalState(final String objectKey) throws IOException { + // get local state if it exists + final File persistenceFile = getPersistenceFile(); + + if (persistenceFile.exists()) { + final Properties props = new Properties(); + try (final FileInputStream fis = new FileInputStream(persistenceFile)) { + props.load(fis); + } catch (IOException ioe) { + getLogger().warn("Failed to recover local state for {} due to {}. Assuming no local state and " + + "restarting upload.", objectKey, ioe.getMessage()); + return null; + } + if (props.containsKey(objectKey)) { + final String localSerialState = props.getProperty(objectKey); + if (localSerialState != null) { + try { + return MultipartState.newMultipartState(localSerialState); + } catch (final RuntimeException rte) { + getLogger().warn("Failed to recover local state for {} due to corrupt data in state.", objectKey, rte.getMessage()); + return null; + } + } + } + } + return null; + } + + public synchronized void persistLocalState(final String objectKey, final MultipartState currState) throws IOException { + JsonMapper mapper = new JsonMapper(); + final String currStateStr = (currState == null) ? null : mapper.writeValueAsString(currState); + final File persistenceFile = getPersistenceFile(); + final File parentDir = persistenceFile.getParentFile(); + if (!parentDir.exists() && !parentDir.mkdirs()) { + throw new IOException("Persistence directory (" + parentDir.getAbsolutePath() + ") does not exist and " + + "could not be created."); + } + final Properties props = new Properties(); + if (persistenceFile.exists()) { + try (final FileInputStream fis = new FileInputStream(persistenceFile)) { + props.load(fis); + } + } + if (currStateStr != null) { + currState.setTimestamp(System.currentTimeMillis()); + props.setProperty(objectKey, currStateStr); + } else { + props.remove(objectKey); + } + + if (props.size() > 0) { + try (final FileOutputStream fos = new FileOutputStream(persistenceFile)) { + props.store(fos, null); + } catch (IOException ioe) { + getLogger().error("Could not store state {} due to {}.", + persistenceFile.getAbsolutePath(), ioe.getMessage()); + } + } else { + if (persistenceFile.exists()) { + try { + Files.delete(persistenceFile.toPath()); + } catch (IOException ioe) { + getLogger().error("Could not remove state file {} due to {}.", + persistenceFile.getAbsolutePath(), ioe.getMessage()); + } + } + } + } + + protected synchronized void removeLocalState(final String objectKey) throws IOException { + persistLocalState(objectKey, null); + } + + // Clear the multi-segment upload status information in the local cache + private synchronized void ageOffLocalState(long ageCutoff) { + // get local state if it exists + final File persistenceFile = getPersistenceFile(); + if (!persistenceFile.exists()) { + return; + } + Properties props = new Properties(); + try (final FileInputStream fis = new FileInputStream(persistenceFile)) { + props.load(fis); + } catch (final IOException ioe) { + getLogger().warn("Failed to ageoff remove local state due to {}", + ioe.getMessage()); + return; + } + for (Entry entry : props.entrySet()) { + final String key = (String) entry.getKey(); + final String localSerialState = props.getProperty(key); + if (StringUtils.isBlank(localSerialState)) { + continue; + } + final MultipartState state; + try { + state = MultipartState.newMultipartState(localSerialState); + } catch (JsonProcessingException e) { + getLogger().warn("Failed to recover local state for {} due to corrupt data in state.", key, e.getMessage()); + continue; + } + if (state.getTimestamp() < ageCutoff) { + getLogger().warn("Removing local state for {} due to exceeding ageoff time", + key); + try { + removeLocalState(key); + } catch (final IOException ioe) { + getLogger().warn("Failed to remove local state for {} due to {}", + key, ioe.getMessage()); + + } + } + } + } + + + private Map doUpload(CustomProperties customProperties, InputStream rawIn, ObsClient obsClient, ProcessContext context) throws IOException { + Map attributes = new HashMap<>(); + try (final InputStream in = new BufferedInputStream(rawIn)) { + customProperties.setContent(in); + final ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setContentLength(customProperties.getFlowFile().getSize()); + if (StringUtils.isNoneBlank(customProperties.getContentType())) { + objectMetadata.setContentType(customProperties.getContentType()); + attributes.put(OBS_CONTENT_TYPE, customProperties.getContentType()); + } + if (StringUtils.isNoneBlank(customProperties.getCacheControl())) { + objectMetadata.setCacheControl(customProperties.getCacheControl()); + attributes.put(OBS_CACHE_CONTROL, customProperties.getCacheControl()); + } + + final String contentDisposition = customProperties.getContentDisposition(); + String dispositionValue = null; + if (CONTENT_DISPOSITION_INLINE.equals(contentDisposition)) { + dispositionValue = contentDisposition; + attributes.put(OBS_CONTENT_DISPOSITION, contentDisposition); + } else if (CONTENT_DISPOSITION_ATTACHMENT.equals(contentDisposition)) { + dispositionValue = CONTENT_DISPOSITION_ATTACHMENT + "; filename=\"" + dispositionValue + "\""; + attributes.put(OBS_CONTENT_DISPOSITION, dispositionValue); + } else if (CONTENT_DISPOSITION_FILENAME.equals(contentDisposition)) { + dispositionValue = URLEncoder.encode(customProperties.getFileName(), "UTF-8"); + } + if (StringUtils.isNotBlank(dispositionValue)) { + objectMetadata.setContentDisposition(dispositionValue); + } + Map userMetaData = buildUserMetaData(context, customProperties); + if (MapUtils.isNotEmpty(userMetaData)) { + objectMetadata.setUserMetadata(userMetaData); + } + // Whether to encrypt it and how to encrypt it + attributes.put(OBS_ENCRYPTION_STRATEGY, customProperties.getEncryptionService().getStrategyName()); + if (ObsServiceEncryptionService.STRATEGY_NAME_SSE_KMS.equals(customProperties.getEncryptionService().getStrategyName())) { + // sseKms + attributes.put(OBS_SSE_ALGORITHM, SSEAlgorithmEnum.KMS.getCode()); + } else if (ObsServiceEncryptionService.STRATEGY_NAME_SSE_C.equals(customProperties.getEncryptionService().getStrategyName())) { + // sseC + attributes.put(OBS_SSE_ALGORITHM, SSEAlgorithmEnum.AES256.getCode()); + } + if (customProperties.getFlowFile().getSize() <= customProperties.getMultipartThreshold()) { + singleUpload(customProperties, attributes, context, objectMetadata, obsClient); + } else { + multipartUpload(customProperties, attributes, context, objectMetadata, obsClient); + } + } + return attributes; + } + + private void singleUpload(final CustomProperties customProperties, Map attributes, ProcessContext context, ObjectMetadata objectMetadata, ObsClient obsClient) { + + final PutObjectRequest request = new PutObjectRequest(customProperties.getBucket(), customProperties.getObjectKey(), customProperties.getContent()); + request.setMetadata(objectMetadata); + customProperties.getEncryptionService().configurePutObjectRequest(request, objectMetadata); + request.getMetadata().setObjectStorageClass(customProperties.getStorageClass()); + // 设置acl + setAcl(customProperties, request, context); + try { + final PutObjectResult result = obsClient.putObject(request); + PutResult putResult = new PutResult(); + putResult.setBucketName(result.getBucketName()); + putResult.setETag(result.getEtag()); + putResult.setObjectUrl(result.getObjectUrl()); + putResult.setStorageClass(result.getObjectStorageClass()); + putResult.setVersionId(result.getVersionId()); + putResult.setObjectKey(result.getObjectKey()); + putResult.setPutType(OBS_API_METHOD_PUT_OBJECT); + attributes.putAll(getResultAttributes(putResult, objectMetadata)); + } catch (Exception e) { + getLogger().info("Failure completing upload flowfile={} bucket={} key={} reason={}", + customProperties.getFileName(), customProperties.getBucket(), customProperties.getObjectKey(), e.getMessage()); + throw (e); + } + } + + private void multipartUpload(final CustomProperties customProperties, Map attributes, ProcessContext context, ObjectMetadata objectMetadata, ObsClient obsClient) throws IOException { + final String bucket = customProperties.getBucket(); + final String objectKey = customProperties.getObjectKey(); + final String fileName = customProperties.getFileName(); + + MultipartState currentState = initiateMultipartUpload(customProperties, objectMetadata, obsClient, context); + attributes.put(OBS_UPLOAD_ID, currentState.getUploadId()); + // upload the rest parts + uploadAllParts(currentState, customProperties, objectMetadata, obsClient); + + // complete multipart upload + CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest( + bucket, objectKey, currentState.getUploadId(), currentState.getPartETags()); + + // No call to an encryption service is needed for a CompleteMultipartUploadRequest. + try { + CompleteMultipartUploadResult completeResult = obsClient.completeMultipartUpload(completeRequest); + getLogger().info("Success completing upload flowfile={} etag={} uploadId={}", + fileName, completeResult.getEtag(), currentState.getUploadId()); + PutResult putResult = new PutResult(); + putResult.setBucketName(completeResult.getBucketName()); + putResult.setETag(completeResult.getEtag()); + putResult.setObjectUrl(completeResult.getObjectUrl()); + putResult.setStorageClass(currentState.getStorageClass()); + putResult.setVersionId(completeResult.getVersionId()); + putResult.setObjectKey(completeResult.getObjectKey()); + putResult.setPutType(OBS_API_METHOD_MULTIPART_UPLOAD); + attributes.putAll(getResultAttributes(putResult, objectMetadata)); + } catch (Exception e) { + getLogger().info("Failure completing upload flowfile={} bucket={} key={} reason={}", + fileName, bucket, objectKey, e.getMessage()); + throw (e); + } + } + + public MultipartState initiateMultipartUpload(CustomProperties customProperties, ObjectMetadata objectMetadata, ObsClient obsClient, ProcessContext context) throws IOException { + final String fileName = customProperties.getFileName(); + final String bucket = customProperties.getBucket(); + final String objectKey = customProperties.getObjectKey(); + + // multipart upload + // load or create persistent state + MultipartState currentState = getState(obsClient, customProperties); + // initiate multipart upload or find position in file + if (StringUtils.isBlank(currentState.getUploadId())) { + final InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest(bucket, objectKey); + initiateRequest.setMetadata(objectMetadata); + customProperties.getEncryptionService().configureInitiateMultipartUploadRequest(initiateRequest, objectMetadata); + initiateRequest.getMetadata().setObjectStorageClass(customProperties.getStorageClass()); + // 设置acl + setAcl(customProperties, initiateRequest, context); + try { + final InitiateMultipartUploadResult initiateResult = obsClient.initiateMultipartUpload(initiateRequest); + currentState.setUploadId(initiateResult.getUploadId()); + currentState.getPartETags().clear(); + try { + persistLocalState(customProperties.getCacheKey(), currentState); + } catch (Exception e) { + getLogger().info("Exception saving cache state while processing flow file: " + + e.getMessage()); + throw (new ProcessException("Exception saving cache state", e)); + } + getLogger().info("Success initiating upload flowfile={} available={} position={} length={} bucket={} key={} uploadId={}", + fileName, customProperties.getContent().available(), currentState.getFilePosition(), + currentState.getContentLength(), bucket, objectKey, + currentState.getUploadId()); + + } catch (Exception e) { + getLogger().info("Failure initiating upload flowfile={} bucket={} key={} reason={}", + customProperties.getFileName(), customProperties.getBucket(), customProperties.getObjectKey(), e.getMessage()); + throw (e); + } + } else { + if (currentState.getFilePosition() > 0) { + try { + final long skipped = customProperties.getContent().skip(currentState.getFilePosition()); + if (skipped != currentState.getFilePosition()) { + getLogger().info("Failure skipping to resume upload flowfile={} bucket={} key={} position={} skipped={}", + customProperties, bucket, objectKey, + currentState.getFilePosition(), skipped); + } + } catch (Exception e) { + getLogger().info("Failure skipping to resume upload flowfile={} bucket={} key={} position={} reason={}", + fileName, bucket, objectKey, currentState.getFilePosition(), + e.getMessage()); + throw (new ProcessException(e)); + } + } + } + return currentState; + } + + private void uploadAllParts(MultipartState currentState, CustomProperties customProperties, ObjectMetadata objectMetadata, ObsClient obsClient) throws IOException { + // upload parts + long thisPartSize; + for (int part = currentState.getPartETags().size() + 1; + currentState.getFilePosition() < currentState.getContentLength(); part++) { + if (!PutOBSObject.this.isScheduled()) { + throw new IOException(OBS_PROCESS_UNSCHEDULED_MESSAGE + " flowfile=" + customProperties.getFileName() + " part=" + part + " uploadId=" + currentState.getUploadId()); + } + boolean isLastPart = (currentState.getContentLength() - currentState.getFilePosition()) <= currentState.getPartSize(); + thisPartSize = isLastPart ? currentState.getContentLength() - currentState.getFilePosition() : currentState.getPartSize(); + UploadPartRequest uploadRequest = new UploadPartRequest(customProperties.getBucket(), customProperties.getObjectKey()); + uploadRequest.setUploadId(currentState.getUploadId()); + uploadRequest.setInput(customProperties.getContent()); + uploadRequest.setPartNumber(part); + uploadRequest.setPartSize(thisPartSize); + // The last section of the file is set to automatically close the stream + uploadRequest.setAutoClose(isLastPart); + customProperties.getEncryptionService().configureUploadPartRequest(uploadRequest, objectMetadata); + try { + UploadPartResult uploadPartResult = obsClient.uploadPart(uploadRequest); + currentState.addPartETag(new PartEtag(uploadPartResult.getEtag(), uploadPartResult.getPartNumber())); + currentState.setFilePosition(currentState.getFilePosition() + thisPartSize); + try { + persistLocalState(customProperties.getCacheKey(), currentState); + } catch (Exception e) { + getLogger().info("Exception saving cache state processing flow file: " + + e.getMessage()); + } + int available = 0; + try { + available = customProperties.getContent().available(); + } catch (IOException e) { + // in case of the last part, the stream is already closed + } + getLogger().info("Success uploading part flowfile={} part={} available={} " + + "etag={} uploadId={}", customProperties.getFileName(), part, available, + uploadPartResult.getEtag(), currentState.getUploadId()); + if (available == 0) { + break; + } + } catch (Exception e) { + getLogger().info("Failure uploading part flowfile={} part={} bucket={} key={} " + + "reason={}", customProperties.getFileName(), part, customProperties.getBucket(), customProperties.getObjectKey(), e.getMessage()); + throw (e); + } + } + } + + // Set the Acl, with finer grained permission Settings taking precedence + private void setAcl(CustomProperties customProperties, PutObjectBasicRequest request, ProcessContext context) { + // Predefined permission policies have a low priority and may be overwritten + final AccessControlList cannedAcl = createCannedACL(context, customProperties.getFlowFile()); + if (cannedAcl != null) { + request.setAcl(cannedAcl); + } + final AccessControlList acl = createACL(context, customProperties.getFlowFile()); + if (acl != null) { + request.setAcl(acl); + } + } + + private Map buildUserMetaData(final ProcessContext context, final CustomProperties customProperties) { + final Map userMetadata = new HashMap<>(); + for (final Entry entry : context.getProperties().entrySet()) { + if (entry.getKey().isDynamic()) { + final String value = context.getProperty( + entry.getKey()).evaluateAttributeExpressions(customProperties.getFlowFile()).getValue(); + userMetadata.put(entry.getKey().getName(), value); + } + } + return userMetadata; + } + + private Map getResultAttributes(PutResult putResult, ObjectMetadata objectMetadata) { + Map attributes = new HashMap<>(); + if (putResult.getVersionId() != null) { + attributes.put(OBS_VERSION, putResult.getVersionId()); + } + if (putResult.getETag() != null) { + attributes.put(OBS_E_TAG, putResult.getETag()); + } + attributes.put(OBS_STORAGE_CLASS, (putResult.getStorageClass() == null ? StorageClassEnum.STANDARD : putResult.getStorageClass()).getCode()); + if (objectMetadata.getAllMetadata().size() > 0) { + StringBuilder userMetaBuilder = new StringBuilder(); + for (String userKey : objectMetadata.getAllMetadata().keySet()) { + userMetaBuilder.append(userKey).append("=").append(objectMetadata.getAllMetadata().get(userKey)); + } + attributes.put(OBS_USER_META, userMetaBuilder.toString()); + } + attributes.put(OBS_API_METHOD_ATTR_KEY, putResult.getPutType()); + attributes.put(OBJECT_URL, putResult.getObjectUrl()); + return attributes; + } + + private MultipartState getState(ObsClient obsClient, CustomProperties customProperties) throws IOException { + try { + MultipartState currentState = getLocalStateIfInOBS(obsClient, customProperties.getBucket(), customProperties.getCacheKey()); + + if (currentState != null) { + if (currentState.getPartETags().size() > 0) { + final PartEtag lastETag = currentState.getPartETags().get( + currentState.getPartETags().size() - 1); + getLogger().info("Resuming upload for flowfile='{}' bucket='{}' key='{}' " + + "uploadID='{}' filePosition='{}' partSize='{}' storageClass='{}' " + + "contentLength='{}' partsLoaded={} lastPart={}/{}", + customProperties.getFileName(), customProperties.getBucket(), customProperties.getObjectKey(), currentState.getUploadId(), + currentState.getFilePosition(), currentState.getPartSize(), + currentState.getStorageClass(), + currentState.getContentLength(), + currentState.getPartETags().size(), + Integer.toString(lastETag.getPartNumber()), + lastETag.getEtag()); + } else { + getLogger().info("Resuming upload for flowfile='{}' bucket='{}' key='{}' " + + "uploadID='{}' filePosition='{}' partSize='{}' storageClass='{}' " + + "contentLength='{}' no partsLoaded", + customProperties.getFileName(), customProperties.getBucket(), customProperties.getObjectKey(), currentState.getUploadId(), + currentState.getFilePosition(), currentState.getPartSize(), + currentState.getStorageClass().toString(), + currentState.getContentLength()); + } + } else { + currentState = new MultipartState(); + currentState.setPartSize(customProperties.getMultipartPartSize()); + currentState.setStorageClass(customProperties.getStorageClass()); + currentState.setContentLength(customProperties.getFlowFile().getSize()); + // Save the current upload status + persistLocalState(customProperties.getCacheKey(), currentState); + getLogger().info("Starting new upload for flowfile='{}' bucket='{}' key='{}'", + customProperties.getFileName(), customProperties.getBucket(), customProperties.getObjectKey()); + } + return currentState; + } catch (IOException e) { + getLogger().error("IOException initiating cache state while processing flow files: " + + e.getMessage()); + throw (e); + } + + } + + private final Lock bucketLock = new ReentrantLock(); + private final AtomicLong lastAgeOff = new AtomicLong(0L); + private final DateFormat logFormat = new SimpleDateFormat(); + + protected void ageOffUploads(final ProcessContext context, final ObsClient obsClient, final long now, String bucket) { + MultipartUploadListing oldUploads = getAgeOffListAndAgeOffLocalState(context, obsClient, now, bucket); + for (MultipartUpload upload : oldUploads.getMultipartTaskList()) { + abortMultipartUpload(obsClient, oldUploads.getBucketName(), upload); + } + } + + public MultipartUploadListing getAgeOffListAndAgeOffLocalState(final ProcessContext context, final ObsClient obsClient, final long now, String bucket) { + final long ageOffInterval = context.getProperty(MULTIPART_AGEOFF_INTERVAL).asTimePeriod(TimeUnit.MILLISECONDS); + final Long maxAge = context.getProperty(MULTIPART_MAX_AGE).asTimePeriod(TimeUnit.MILLISECONDS); + final long ageCutOff = now - maxAge; + + final List ageOffList = new ArrayList<>(); + // Uncompleted multisegment upload tasks are cleared every 60 minutes (by default) + if ((lastAgeOff.get() < now - ageOffInterval) && bucketLock.tryLock()) { + try { + ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest(bucket); + MultipartUploadListing listing = obsClient.listMultipartUploads(listRequest); + // All the multi-segment upload tasks under the bucket are traversed, and those that exceed the maximum threshold are recorded + for (MultipartUpload upload : listing.getMultipartTaskList()) { + long firstUploadTime = upload.getInitiatedDate().getTime(); + if (firstUploadTime < ageCutOff) { + ageOffList.add(upload); + } + } + // ageOff any local state + ageOffLocalState(ageCutOff); + // Update timestamp + lastAgeOff.set(System.currentTimeMillis()); + } catch (Exception e) { + if (e instanceof ObsException + && ((ObsException) e).getResponseCode() == 403 + && ((ObsException) e).getErrorCode().equals("AccessDenied")) { + getLogger().warn("AccessDenied checking OBS Multipart Upload list for {}: {} " + + "** The configured user does not have the OBS:ListBucketMultipartUploads permission " + + "for this bucket, OBS ageoff cannot occur without this permission. Next ageoff check " + + "time is being advanced by interval to prevent checking on every upload **", + bucket, e.getMessage()); + lastAgeOff.set(System.currentTimeMillis()); + } else { + getLogger().error("Error checking OBS Multipart Upload list for {}: {}", + bucket, e.getMessage()); + } + } finally { + bucketLock.unlock(); + } + } + return new MultipartUploadListing.Builder().bucketName(bucket).multipartTaskList(ageOffList).builder(); + } + + public void abortMultipartUpload(final ObsClient obsClient, final String bucket, final MultipartUpload upload) { + final String uploadKey = upload.getObjectKey(); + final String uploadId = upload.getUploadId(); + final AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest( + bucket, uploadKey, uploadId); + // No call to an encryption service is necessary for an AbortMultipartUploadRequest. + try { + obsClient.abortMultipartUpload(abortRequest); + getLogger().info("Aborting out of date multipart upload, bucket {} key {} ID {}, initiated {}", + bucket, uploadKey, uploadId, logFormat.format(upload.getInitiatedDate())); + } catch (Exception ace) { + getLogger().info("Error trying to abort multipart upload from bucket {} with key {} and ID {}: {}", + bucket, uploadKey, uploadId, ace.getMessage()); + } + } + + public static class MultipartState implements Serializable { + + private static final long serialVersionUID = 9006072180563519740L; + private String _uploadId; + private Long _filePosition; + private final List _partETags; + private Long _partSize; + private StorageClassEnum _storageClassEnum; + private Long _contentLength; + private Long _timestamp; + + public MultipartState() { + _uploadId = ""; + _filePosition = 0L; + _partETags = new ArrayList<>(); + _partSize = 0L; + _storageClassEnum = StorageClassEnum.STANDARD; + _contentLength = 0L; + _timestamp = System.currentTimeMillis(); + } + + public static MultipartState newMultipartState(String json) throws JsonProcessingException { + JsonMapper jsonMapper = new JsonMapper(); + return jsonMapper.readValue(json, MultipartState.class); + } + + public String getUploadId() { + return _uploadId; + } + + public void setUploadId(String id) { + _uploadId = id; + } + + public Long getFilePosition() { + return _filePosition; + } + + public void setFilePosition(Long pos) { + _filePosition = pos; + } + + public List getPartETags() { + return _partETags; + } + + public void addPartETag(PartEtag tag) { + _partETags.add(tag); + } + + public Long getPartSize() { + return _partSize; + } + + public void setPartSize(Long size) { + _partSize = size; + } + + public StorageClassEnum getStorageClass() { + return _storageClassEnum; + } + + public void setStorageClass(StorageClassEnum storageClassEnum) { + _storageClassEnum = storageClassEnum; + } + + public Long getContentLength() { + return _contentLength; + } + + public void setContentLength(Long length) { + _contentLength = length; + } + + public Long getTimestamp() { + return _timestamp; + } + + public void setTimestamp(Long timestamp) { + _timestamp = timestamp; + } + } + + public static class PutResult { + private String bucketName; + + private String objectKey; + + private String eTag; + + private String versionId; + + private StorageClassEnum storageClass; + + private String objectUrl; + + private String location; + + private String encodingType; + + private InputStream callbackResponseBody; + + public String getETag() { + return eTag; + } + + public void setETag(String eTag) { + this.eTag = eTag; + } + + public String getPutType() { + return putType; + } + + public void setPutType(String putType) { + this.putType = putType; + } + + private String putType; + + public String getBucketName() { + return bucketName; + } + + public void setBucketName(String bucketName) { + this.bucketName = bucketName; + } + + public String getObjectKey() { + return objectKey; + } + + public void setObjectKey(String objectKey) { + this.objectKey = objectKey; + } + + public String getVersionId() { + return versionId; + } + + public void setVersionId(String versionId) { + this.versionId = versionId; + } + + public StorageClassEnum getStorageClass() { + return storageClass; + } + + public void setStorageClass(StorageClassEnum storageClass) { + this.storageClass = storageClass; + } + + public String getObjectUrl() { + return objectUrl; + } + + public void setObjectUrl(String objectUrl) { + this.objectUrl = objectUrl; + } + + public String getLocation() { + return location; + } + + public void setLocation(String location) { + this.location = location; + } + + public String getEncodingType() { + return encodingType; + } + + public void setEncodingType(String encodingType) { + this.encodingType = encodingType; + } + + public InputStream getCallbackResponseBody() { + return callbackResponseBody; + } + + public void setCallbackResponseBody(InputStream callbackResponseBody) { + this.callbackResponseBody = callbackResponseBody; + } + } + + /** + * 输入属性 + */ + public static class CustomProperties { + private String bucket; + private String objectKey; + private String cacheKey; + private String fileName; + private Long multipartThreshold; + private Long multipartPartSize; + private String contentType; + private String cacheControl; + private ObsServiceEncryptionService encryptionService; + private FlowFile flowFile; + private InputStream content; + private StorageClassEnum storageClass; + + public StorageClassEnum getStorageClass() { + return storageClass; + } + + public void setStorageClass(StorageClassEnum storageClass) { + this.storageClass = storageClass; + } + + public FlowFile getFlowFile() { + return flowFile; + } + + public void setFlowFile(FlowFile flowFile) { + this.flowFile = flowFile; + } + + public InputStream getContent() { + return content; + } + + public void setContent(InputStream content) { + this.content = content; + } + + public String getCacheKey() { + return cacheKey; + } + + public void setCacheKey(String cacheKey) { + this.cacheKey = cacheKey; + } + + public String getContentType() { + return contentType; + } + + public void setContentType(String contentType) { + this.contentType = contentType; + } + + public String getCacheControl() { + return cacheControl; + } + + public void setCacheControl(String cacheControl) { + this.cacheControl = cacheControl; + } + + public ObsServiceEncryptionService getEncryptionService() { + return encryptionService; + } + + public void setEncryptionService(ObsServiceEncryptionService encryptionService) { + this.encryptionService = encryptionService; + } + + public String getContentDisposition() { + return contentDisposition; + } + + public void setContentDisposition(String contentDisposition) { + this.contentDisposition = contentDisposition; + } + + private String contentDisposition; + + public String getBucket() { + return bucket; + } + + public void setBucket(String bucket) { + this.bucket = bucket; + } + + public String getObjectKey() { + return objectKey; + } + + public void setObjectKey(String objectKey) { + this.objectKey = objectKey; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public Long getMultipartThreshold() { + return multipartThreshold; + } + + public void setMultipartThreshold(Long multipartThreshold) { + this.multipartThreshold = multipartThreshold; + } + + public Long getMultipartPartSize() { + return multipartPartSize; + } + + public void setMultipartPartSize(Long multipartPartSize) { + this.multipartPartSize = multipartPartSize; + } + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/NoOpEncryptionStrategy.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/NoOpEncryptionStrategy.java new file mode 100644 index 0000000000000000000000000000000000000000..0d6f23fe5a320d30796490c46c3e191d186d2160 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/NoOpEncryptionStrategy.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.obs.encryption; + +public class NoOpEncryptionStrategy implements OBSEncryptionStrategy { +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/OBSEncryptionStrategy.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/OBSEncryptionStrategy.java new file mode 100644 index 0000000000000000000000000000000000000000..1018c7bf57b1893d0528894397b62603757fe5a4 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/OBSEncryptionStrategy.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.obs.encryption; + +import com.obs.services.model.*; +import org.apache.nifi.components.ValidationResult; + +/** + * This interface defines the API for OBS encryption strategies. The methods have empty defaults + * to minimize the burden on implementations. + * + */ +public interface OBSEncryptionStrategy { + + /** + * Configure a {@link PutObjectRequest} for encryption. + * @param request the request to configure. + * @param keyValue the key id or key material. + */ + default void configurePutObjectRequest(PutObjectRequest request, String keyValue) { + } + + /** + * Configure an {@link InitiateMultipartUploadRequest} for encryption. + * @param request the request to configure. + * @param keyValue the key id or key material. + */ + default void configureInitiateMultipartUploadRequest(InitiateMultipartUploadRequest request, String keyValue) { + } + + /** + * Configure a {@link GetObjectRequest} for encryption. + * @param request the request to configure. + * @param keyValue the key id or key material. + */ + default void configureGetObjectRequest(GetObjectRequest request, String keyValue) { + } + + /** + * Configure an {@link UploadPartRequest} for encryption. + * @param request the request to configure. + * @param keyValue the key id or key material. + */ + default void configureUploadPartRequest(UploadPartRequest request, String keyValue) { + } + + /** + * Validate a key id or key material. + * + * @param keyValue key id or key material to validate. + * @return ValidationResult instance. + */ + default ValidationResult validateKey(String keyValue) { + return new ValidationResult.Builder().valid(true).build(); + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/ServerSideCEncryptionStrategy.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/ServerSideCEncryptionStrategy.java new file mode 100644 index 0000000000000000000000000000000000000000..d1ec8a12951f10f092742f5cf668210623809e96 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/ServerSideCEncryptionStrategy.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.obs.encryption; + +import com.obs.services.model.*; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.components.ValidationResult; + +/** + * This strategy uses a customer key to perform server-side encryption. Use this strategy when you want the server to perform the encryption, + * (meaning you pay cost of processing) and when you want to manage the key material yourself. + */ +public class ServerSideCEncryptionStrategy implements OBSEncryptionStrategy { + @Override + public void configurePutObjectRequest(PutObjectRequest request, String keyValue) { + SseCHeader sseCHeader = new SseCHeader(); + sseCHeader.setSseCKeyBase64(keyValue); + request.setSseCHeader(sseCHeader); + } + + @Override + public void configureInitiateMultipartUploadRequest(InitiateMultipartUploadRequest request, String keyValue) { + SseCHeader sseCHeader = new SseCHeader(); + sseCHeader.setSseCKeyBase64(keyValue); + request.setSseCHeader(sseCHeader); + } + + @Override + public void configureGetObjectRequest(GetObjectRequest request, String keyValue) { + SseCHeader sseCHeader = new SseCHeader(); + sseCHeader.setSseCKeyBase64(keyValue); + request.setSseCHeader(sseCHeader); + } + + @Override + public void configureUploadPartRequest(UploadPartRequest request, String keyValue) { + SseCHeader sseCHeader = new SseCHeader(); + sseCHeader.setSseCKeyBase64(keyValue); + request.setSseCHeader(sseCHeader); + } + + @Override + public ValidationResult validateKey(String keyValue) { + if (StringUtils.isBlank(keyValue)) { + return new ValidationResult.Builder() + .subject("Key Material") + .valid(false) + .explanation("it is empty") + .build(); + } + return new ValidationResult.Builder().valid(true).build(); + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/ServerSideKMSEncryptionStrategy.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/ServerSideKMSEncryptionStrategy.java new file mode 100644 index 0000000000000000000000000000000000000000..0cb69ee25020205175f74b0b9bf31ca6c1be650f --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/ServerSideKMSEncryptionStrategy.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.obs.encryption; + +import com.obs.services.model.InitiateMultipartUploadRequest; +import com.obs.services.model.PutObjectRequest; +import com.obs.services.model.SseKmsHeader; + +/** + * This strategy uses a KMS key to perform server-side encryption. Use this strategy when you want the server to perform the encryption, + * (meaning you pay the cost of processing) and when you want to use a KMS key. + */ +public class ServerSideKMSEncryptionStrategy implements OBSEncryptionStrategy { + @Override + public void configurePutObjectRequest(PutObjectRequest request, String keyValue) { + SseKmsHeader sseKmsHeader = new SseKmsHeader(); + sseKmsHeader.setKmsKeyId(keyValue); + request.setSseKmsHeader(sseKmsHeader); + } + @Override + public void configureInitiateMultipartUploadRequest(InitiateMultipartUploadRequest request, String keyValue) { + SseKmsHeader sseKmsHeader = new SseKmsHeader(); + sseKmsHeader.setKmsKeyId(keyValue); + request.setSseKmsHeader(sseKmsHeader); + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/StandardOBSEncryptionService.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/StandardOBSEncryptionService.java new file mode 100644 index 0000000000000000000000000000000000000000..94d9634ff08a8b3131c39e988e749a597a11b1de --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/encryption/StandardOBSEncryptionService.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.huawei.obs.encryption; + +import com.obs.services.model.*; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnEnabled; +import org.apache.nifi.components.*; +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.processors.huawei.obs.OBSRegions; +import org.apache.nifi.processors.huawei.obs.ObsServiceEncryptionService; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + + +@Tags({"service", "huaweiCloud", "OBS", "encryption", "encrypt", "decryption", "decrypt", "key"}) +@CapabilityDescription("Adds configurable encryption to OBS Put and OBS Fetch operations.") +public class StandardOBSEncryptionService extends AbstractControllerService implements ObsServiceEncryptionService { + private static final Logger logger = LoggerFactory.getLogger(StandardOBSEncryptionService.class); + + private static final Map NAMED_STRATEGIES = new HashMap() {{ + put(STRATEGY_NAME_NONE, new NoOpEncryptionStrategy()); + put(STRATEGY_NAME_SSE_KMS, new ServerSideKMSEncryptionStrategy()); + put(STRATEGY_NAME_SSE_C, new ServerSideCEncryptionStrategy()); + }}; + + public static final AllowableValue NONE = new AllowableValue(STRATEGY_NAME_NONE, "None","No encryption."); + public static final AllowableValue SSE_KMS = new AllowableValue(STRATEGY_NAME_SSE_KMS, "Server-side KMS","The key that uses server-side KMS encryption needs to be in the same area as the bucket."); + public static final AllowableValue SSE_C = new AllowableValue(STRATEGY_NAME_SSE_C, "Server-side Customer Key","The encryption is performed using the key provided by the server and the client. The key must be base64 encoded."); + + public static final Map ENCRYPTION_STRATEGY_ALLOWABLE_VALUES = new HashMap() {{ + put(STRATEGY_NAME_NONE, NONE); + put(STRATEGY_NAME_SSE_KMS, SSE_KMS); + put(STRATEGY_NAME_SSE_C, SSE_C); + }}; + + public static final PropertyDescriptor ENCRYPTION_STRATEGY = new PropertyDescriptor.Builder() + .name("encryption-strategy") + .displayName("Encryption Strategy") + .description("Strategy to use for OBS data encryption and decryption.") + .allowableValues(NONE, SSE_KMS, SSE_C) + .required(true) + .defaultValue(NONE.getValue()) + .build(); + + public static final PropertyDescriptor ENCRYPTION_VALUE = new PropertyDescriptor.Builder() + .name("key-id-or-key-material") + .displayName("Key ID or Key Material") + .description("For None and Server-side OBS: not used. For Server-side KMS: the KMS Key ID must be configured. " + + "For Server-side Customer Key: the Key Material must be specified in Base64 encoded form. " + + "In case of Server-side Customer Key, the key must be an AES-256 key") + .required(false) + .sensitive(true) + .addValidator((subject, input, context) -> new ValidationResult.Builder().valid(true).build()) // will be validated in customValidate() + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .build(); + + public static final PropertyDescriptor KMS_REGION = new PropertyDescriptor.Builder() + .name("kms-region") + .displayName("KMS Region") + .description("The Region of the HuaweiCloud Key Management Service. Only used in case of Server-side KMS.") + .required(false) + .allowableValues(OBSRegions.getAvailableOBSRegions()) + .defaultValue(OBSRegions.DEFAULT_REGION.getName()) + .build(); + + private String keyValue = ""; + private String kmsRegion = ""; + private OBSEncryptionStrategy encryptionStrategy = new NoOpEncryptionStrategy(); + private String strategyName = STRATEGY_NAME_NONE; + + @OnEnabled + public void onConfigured(final ConfigurationContext context) throws InitializationException { + final String newStrategyName = context.getProperty(ENCRYPTION_STRATEGY).getValue(); + final String newKeyValue = context.getProperty(ENCRYPTION_VALUE).evaluateAttributeExpressions().getValue(); + final OBSEncryptionStrategy newEncryptionStrategy = NAMED_STRATEGIES.get(newStrategyName); + String newKmsRegion = null; + if (context.getProperty(KMS_REGION) != null ) { + newKmsRegion = context.getProperty(KMS_REGION).getValue(); + } + + if (newEncryptionStrategy == null) { + final String msg = "No encryption strategy found for name: " + strategyName; + logger.warn(msg); + throw new InitializationException(msg); + } + + strategyName = newStrategyName; + encryptionStrategy = newEncryptionStrategy; + keyValue = newKeyValue; + kmsRegion = newKmsRegion; + } + + @Override + protected Collection customValidate(final ValidationContext validationContext) { + Collection validationResults = new ArrayList<>(); + + String encryptionStrategyName = validationContext.getProperty(ENCRYPTION_STRATEGY).getValue(); + String encryptionStrategyDisplayName = ENCRYPTION_STRATEGY_ALLOWABLE_VALUES.get(encryptionStrategyName).getDisplayName(); + PropertyValue encryptionValueProperty = validationContext.getProperty(ENCRYPTION_VALUE); + String encryptionValue = encryptionValueProperty.evaluateAttributeExpressions().getValue(); + + switch (encryptionStrategyName) { + case STRATEGY_NAME_NONE: + if (encryptionValueProperty.isSet()) { + validationResults.add(new ValidationResult.Builder() + .subject(ENCRYPTION_VALUE.getDisplayName()) + .valid(false) + .explanation("the property cannot be specified for encryption strategy " + encryptionStrategyDisplayName) + .build() + ); + } + break; + case STRATEGY_NAME_SSE_KMS: + if (StringUtils.isEmpty(encryptionValue)) { + validationResults.add(new ValidationResult.Builder() + .subject(ENCRYPTION_VALUE.getDisplayName()) + .valid(false) + .explanation("a non-empty Key ID must be specified for encryption strategy " + encryptionStrategyDisplayName) + .build() + ); + } + break; + case STRATEGY_NAME_SSE_C: + if (StringUtils.isEmpty(encryptionValue)) { + validationResults.add(new ValidationResult.Builder() + .subject(ENCRYPTION_VALUE.getDisplayName()) + .valid(false) + .explanation("a non-empty Key Material must be specified for encryption strategy " + encryptionStrategyDisplayName) + .build() + ); + } else { + OBSEncryptionStrategy encryptionStrategy = NAMED_STRATEGIES.get(encryptionStrategyName); + String keyIdOrMaterial = validationContext.getProperty(ENCRYPTION_VALUE).evaluateAttributeExpressions().getValue(); + + validationResults.add(encryptionStrategy.validateKey(keyIdOrMaterial)); + } + break; + } + + return validationResults; + } + + @Override + protected List getSupportedPropertyDescriptors() { + final List properties = new ArrayList<>(); + properties.add(ENCRYPTION_STRATEGY); + properties.add(ENCRYPTION_VALUE); + properties.add(KMS_REGION); + return Collections.unmodifiableList(properties); + } + + @Override + public void configurePutObjectRequest(PutObjectRequest request, ObjectMetadata objectMetadata) { + encryptionStrategy.configurePutObjectRequest(request, keyValue); + } + + @Override + public void configureInitiateMultipartUploadRequest(InitiateMultipartUploadRequest request, ObjectMetadata objectMetadata) { + encryptionStrategy.configureInitiateMultipartUploadRequest(request, keyValue); + } + + @Override + public void configureGetObjectRequest(GetObjectRequest request, ObjectMetadata objectMetadata) { + encryptionStrategy.configureGetObjectRequest(request, keyValue); + } + + @Override + public void configureUploadPartRequest(UploadPartRequest request, ObjectMetadata objectMetadata) { + encryptionStrategy.configureUploadPartRequest(request, keyValue); + } + + @Override + public String getStrategyName() { + return strategyName; + } +} + + diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/model/OBSObjectBucketLister.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/model/OBSObjectBucketLister.java new file mode 100644 index 0000000000000000000000000000000000000000..0c1ad12b597b74bd351156269af4d649a386c32c --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/model/OBSObjectBucketLister.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.huawei.obs.model; + +import com.obs.services.ObsClient; +import com.obs.services.model.*; +import org.apache.nifi.processor.ProcessContext; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; + +public class OBSObjectBucketLister implements ObsBucketLister{ + private final ObsClient client; + private final ListObjectsRequest listObjectsRequest; + private ObjectListing objectListing; + + public OBSObjectBucketLister(ObsClient client, ProcessContext context) { + this.client = client; + final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions().getValue(); + final String delimiter = context.getProperty(DELIMITER).getValue(); + final String prefix = context.getProperty(PREFIX).evaluateAttributeExpressions().getValue(); + listObjectsRequest = new ListObjectsRequest(bucket, prefix, null, delimiter, 0, null); + } + + @Override + public ListVersionsResult listVersions() { + this.objectListing = client.listObjects(listObjectsRequest); + List summaryList = new ArrayList<>(); + for (ObsObject objectSummary : objectListing.getObjects()) { + VersionOrDeleteMarker versionSummary = new VersionOrDeleteMarker.Builder() + .bucketName(objectSummary.getBucketName()) + .lastModified(objectSummary.getMetadata().getLastModified()) + .owner(objectSummary.getOwner()) + .size(objectSummary.getMetadata().getContentLength()) + .storageClass(objectSummary.getMetadata().getObjectStorageClass()) + .isLatest(true) + .key(objectSummary.getObjectKey()).builder(); + summaryList.add(versionSummary); + } + return new ListVersionsResult.Builder().versions(summaryList.toArray(new VersionOrDeleteMarker[0])).builder(); + } + + @Override + public void setNextMarker() { + listObjectsRequest.setMarker(objectListing.getNextMarker()); + } + + @Override + public boolean isTruncated() { + return objectListing != null && objectListing.isTruncated(); + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/model/OBSRecord.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/model/OBSRecord.java new file mode 100644 index 0000000000000000000000000000000000000000..570856e4f56895b2e656670269fefa94bd879fe6 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/model/OBSRecord.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.huawei.obs.model; + +import com.obs.services.model.ObjectMetadata; +import com.obs.services.model.VersionOrDeleteMarker; +import org.apache.nifi.processor.util.list.ListableEntity; +import org.apache.nifi.processors.huawei.obs.OBSUtils; +import org.apache.nifi.serialization.SimpleRecordSchema; +import org.apache.nifi.serialization.record.*; +import org.apache.nifi.serialization.record.Record; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.nifi.processors.huawei.obs.Constants.*; + +public class OBSRecord implements Comparable, Serializable, ListableEntity { + private static final long serialVersionUID = 1L; + private static final RecordSchema SCHEMA; + private String region; + + private String endpoint; + + private VersionOrDeleteMarker versionSummary; + + private ObjectMetadata objectMetadata; + + static { + final List fields = new ArrayList<>(); + fields.add(new RecordField(OBS_OBJECT, RecordFieldType.STRING.getDataType(), false)); + fields.add(new RecordField(OBS_BUCKET, RecordFieldType.STRING.getDataType(), false)); + fields.add(new RecordField(OBS_OWNER, RecordFieldType.STRING.getDataType(), true)); + fields.add(new RecordField(OBS_E_TAG, RecordFieldType.STRING.getDataType(), false)); + fields.add(new RecordField(OBS_LAST_MODIFIED, RecordFieldType.TIMESTAMP.getDataType(), false)); + fields.add(new RecordField(OBS_LENGTH, RecordFieldType.LONG.getDataType(), false)); + fields.add(new RecordField(OBS_STORAGE_CLASS, RecordFieldType.STRING.getDataType(), false)); + fields.add(new RecordField(OBS_IS_LATEST, RecordFieldType.BOOLEAN.getDataType(), false)); + fields.add(new RecordField(OBS_VERSION, RecordFieldType.STRING.getDataType(), true)); + fields.add(new RecordField(OBS_USER_META, RecordFieldType.MAP.getMapDataType(RecordFieldType.STRING.getDataType()), true)); + SCHEMA = new SimpleRecordSchema(fields); + } + + @Override + public Record toRecord() { + final Map values = new HashMap<>(); + values.put(OBS_OBJECT, versionSummary.getKey()); + values.put(OBS_BUCKET, versionSummary.getBucketName()); + if (versionSummary.getOwner() != null) { + values.put(OBS_OWNER, versionSummary.getOwner().getId()); + } + values.put(OBS_E_TAG, versionSummary.getEtag()); + values.put(OBS_LAST_MODIFIED, new Timestamp(versionSummary.getLastModified().getTime())); + values.put(OBS_LENGTH, versionSummary.getSize()); + values.put(OBS_STORAGE_CLASS, versionSummary.getStorageClass()); + values.put(OBS_IS_LATEST, versionSummary.isLatest()); + final String versionId = versionSummary.getVersionId(); + if (versionId != null && !versionId.equals(NULL_VERSION_ID)) { + values.put(OBS_VERSION, versionSummary.getVersionId()); + } + if (objectMetadata != null) { + values.put(OBS_USER_META, objectMetadata.getAllMetadata()); + } + return new MapRecord(SCHEMA, values); + } + + @Override + public int compareTo(OBSRecord o) { + return 0; + } + + @Override + public String getName() { + if (versionSummary == null) { + return null; + } + return versionSummary.getObjectKey(); + } + + @Override + public String getIdentifier() { + if (versionSummary == null) { + return null; + } + return OBSUtils.getUrl(region, endpoint, versionSummary.getBucketName(), versionSummary.getObjectKey()); + } + + @Override + public long getTimestamp() { + if (versionSummary == null) { + return 0; + } + return versionSummary.getLastModified().getTime(); + } + + @Override + public long getSize() { + if (versionSummary == null) { + return 0; + } + return versionSummary.getSize(); + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public VersionOrDeleteMarker getVersionSummary() { + return versionSummary; + } + + public void setVersionSummary(VersionOrDeleteMarker versionSummary) { + this.versionSummary = versionSummary; + } + + public ObjectMetadata getObjectMetadata() { + return objectMetadata; + } + + public void setObjectMetadata(ObjectMetadata objectMetadata) { + this.objectMetadata = objectMetadata; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/model/ObsBucketLister.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/model/ObsBucketLister.java new file mode 100644 index 0000000000000000000000000000000000000000..47a0df606518133038a7d976d35945e139e7a72f --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/obs/model/ObsBucketLister.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.huawei.obs.model; + +import com.obs.services.model.ListVersionsResult; + +public interface ObsBucketLister { + ListVersionsResult listVersions(); + + void setNextMarker(); + + boolean isTruncated(); +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/smn/PublishSMNMessage.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/smn/PublishSMNMessage.java new file mode 100644 index 0000000000000000000000000000000000000000..286275e363f35973035c25d329e0bbff92eda39c --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/smn/PublishSMNMessage.java @@ -0,0 +1,214 @@ +package org.apache.nifi.processors.huawei.smn; + +import com.huaweicloud.sdk.smn.v2.SmnClient; +import com.huaweicloud.sdk.smn.v2.model.PublishMessageRequest; +import com.huaweicloud.sdk.smn.v2.model.PublishMessageRequestBody; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.annotation.behavior.*; +import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.huawei.abstractprocessor.AbstractSMNProcessor; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; + +@SupportsBatching +@InputRequirement(Requirement.INPUT_REQUIRED) +@Tags({"HuaweiCloud", "SMN", "Publish", "Message"}) +@CapabilityDescription("This API is used to publish messages to a topic. After the message ID is returned, " + + "the message has been saved and is to be pushed to the subscribers of the topic.") +public class PublishSMNMessage extends AbstractSMNProcessor { + + public static final PropertyDescriptor SMN_PROJECT_ID = new PropertyDescriptor.Builder() + .name("project_id") + .displayName("Project ID") + .description("Project ID") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .required(true) + .build(); + + public static final PropertyDescriptor SMN_TOPIC_URN = new PropertyDescriptor.Builder() + .name("topic_urn") + .displayName("Topic URN") + .description("The unique resource identifier of the Topic") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .required(true) + .build(); + + public static final PropertyDescriptor SMN_SUBJECT = new PropertyDescriptor.Builder() + .name("subject") + .displayName("Subject") + .description("Specifies the message subject, which is used as the email subject when you publish email messages. " + + "The length of the subject cannot exceed 512 bytes.") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .build(); + + public static final PropertyDescriptor SMN_MESSAGE = new PropertyDescriptor.Builder() + .name("message") + .displayName("Message") + .description("Specifies the message content. The message content must be UTF-8-coded and can be no more than 256 KB.") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor SMN_MESSAGE_STRUCTURE = new PropertyDescriptor.Builder() + .name("message_structure") + .displayName("Message Structure") + .description("Specifies the message structure, which contains JSON strings. " + + "email, sms, http, https, dms, function stage, HMS, APNS, or APNS_SANDBOX are supported.") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor SMN_MESSAGE_TEMPLATE_NAME = new PropertyDescriptor.Builder() + .name("message_template_name") + .displayName("Message Template Name") + .description("Specifies the message template name. To obtain the name, see Querying Message Templates.") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor SMN_TAGS = new PropertyDescriptor.Builder() + .name("tags") + .displayName("Tags") + .description("Specifies the dictionary consisting of variable parameters and values. " + + "This parameter is mandatory when you use a message template to publish messages.") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .dependsOn(SMN_MESSAGE_TEMPLATE_NAME) + .build(); + + public static final PropertyDescriptor SMN_TIME_TO_LIVE = new PropertyDescriptor.Builder() + .name("time_to_live") + .displayName("Time to Live") + .description("Specifies the maximum retention period of a message in SMN. " + + "After the retention period expires, SMN does not send this message.") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final int MAX_SIZE = 256 * 1024; + + private static final List properties = List.of(ACCESS_KEY, SECRET_KEY, SMN_PROJECT_ID, SMN_TOPIC_URN, SMN_REGION, + SMN_SUBJECT, SMN_MESSAGE, + SMN_MESSAGE_STRUCTURE, SMN_MESSAGE_TEMPLATE_NAME, + SMN_TAGS, SMN_TIME_TO_LIVE); + + @Override + public List getSupportedPropertyDescriptors() { + return properties; + } + + @Override + public void onTrigger(final ProcessContext context, final ProcessSession session) { + + FlowFile flowFile = session.get(); + if (flowFile == null) { + + return; + } + + if (flowFile.getSize() > MAX_SIZE) { + + getLogger().error("Cannot publish {} to SMN because its size exceeds Huawei SMN's limit of 256KB; routing to failure", new Object[]{flowFile}); + session.transfer(flowFile, REL_FAILURE); + + return; + } + + try { + + String topicUrn = context.getProperty(SMN_TOPIC_URN).evaluateAttributeExpressions(flowFile).getValue(); + String subject = context.getProperty(SMN_SUBJECT).evaluateAttributeExpressions(flowFile).getValue(); + String message = context.getProperty(SMN_MESSAGE).evaluateAttributeExpressions(flowFile).getValue(); + String messageStructure = context.getProperty(SMN_MESSAGE_STRUCTURE).evaluateAttributeExpressions(flowFile).getValue(); + String messageTemplateName = context.getProperty(SMN_MESSAGE_TEMPLATE_NAME).evaluateAttributeExpressions(flowFile).getValue(); + String tags = context.getProperty(SMN_TAGS).evaluateAttributeExpressions(flowFile).getValue(); + String timeToLive = context.getProperty(SMN_TIME_TO_LIVE).evaluateAttributeExpressions(flowFile).getValue(); + + PublishMessageRequest request = new PublishMessageRequest(); + request.setTopicUrn(topicUrn); + + PublishMessageRequestBody body = new PublishMessageRequestBody(); + body.setSubject(subject); + body.setMessage(message); + body.setMessageStructure(messageStructure); + body.setMessageTemplateName(messageTemplateName); + body.setTimeToLive(timeToLive); + + if (!StringUtils.isEmpty(tags)) { + + Map tagsMap = new ObjectMapper().readValue(tags, HashMap.class); + body.setTags(tagsMap); + } + + request.withBody(body); + SmnClient smnClient = getSmnClient(); + + flowFile = session.putAllAttributes(flowFile, createSmnAttributes(body)); + smnClient.publishMessage(request); + + session.transfer(flowFile, REL_SUCCESS); + session.getProvenanceReporter().send(flowFile, topicUrn); + + getLogger().info("Successfully published notification for {}", new Object[]{flowFile}); + + } catch (Exception e) { + + getLogger().error("Failed to publish message for {} due to {}", new Object[]{flowFile, e}); + + flowFile = session.penalize(flowFile); + session.transfer(flowFile, REL_FAILURE); + } + } + + private Map createSmnAttributes(PublishMessageRequestBody body) { + + Map attributes = new HashMap<>(); + + if (!StringUtils.isEmpty(body.getSubject())) { + + attributes.put(SMN_SUBJECT.getName(), body.getSubject()); + } + + if (!StringUtils.isEmpty(body.getMessage())) { + + attributes.put(SMN_MESSAGE.getName(), body.getMessage()); + } + + if (!StringUtils.isEmpty(body.getMessageStructure())) { + + attributes.put(SMN_MESSAGE_STRUCTURE.getName(), body.getMessageStructure()); + } + + if (!StringUtils.isEmpty(body.getMessageTemplateName())) { + + attributes.put(SMN_MESSAGE_TEMPLATE_NAME.getName(), body.getMessageTemplateName()); + } + + if (!StringUtils.isEmpty(body.getTimeToLive())) { + + attributes.put(SMN_TIME_TO_LIVE.getName(), body.getTimeToLive()); + } + + return attributes; + } +} diff --git a/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/smn/SMNUtils.java b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/smn/SMNUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..58cd6e6ba1ef375ec6f18d5ee58fd6f101761466 --- /dev/null +++ b/nifi-huawei-processors/src/main/java/org/apache/nifi/processors/huawei/smn/SMNUtils.java @@ -0,0 +1,50 @@ +package org.apache.nifi.processors.huawei.smn; + +import com.huaweicloud.sdk.core.auth.BasicCredentials; +import com.huaweicloud.sdk.core.region.Region; +import com.huaweicloud.sdk.smn.v2.region.SmnRegion; +import com.huaweicloud.sdk.smn.v2.SmnClient; +import org.apache.nifi.processor.ProcessContext; + +import java.util.List; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; +import static org.apache.nifi.processors.huawei.smn.PublishSMNMessage.SMN_PROJECT_ID; + +public class SMNUtils { + + public static SmnClient createClient(final ProcessContext processContext) { + + final String accessKey = processContext.getProperty(ACCESS_KEY).evaluateAttributeExpressions().getValue(); + final String secretKey = processContext.getProperty(SECRET_KEY).evaluateAttributeExpressions().getValue(); + final String projectId = processContext.getProperty(SMN_PROJECT_ID).evaluateAttributeExpressions().getValue(); + final String region = processContext.getProperty(SMN_REGION).evaluateAttributeExpressions().getValue(); + + return SmnClient.newBuilder().withCredential( + new BasicCredentials().withAk(accessKey).withSk(secretKey).withProjectId(projectId)) + .withRegion(SmnRegion.valueOf(region)).build(); + } + + public static List getAvailableRegions() { + + return List.of( + SmnRegion.AF_SOUTH_1, + SmnRegion.CN_NORTH_4, + SmnRegion.CN_NORTH_1, + SmnRegion.CN_EAST_2, + SmnRegion.CN_EAST_3, + SmnRegion.CN_SOUTH_1, + SmnRegion.CN_SOUTHWEST_2, + SmnRegion.AP_SOUTHEAST_2, + SmnRegion.AP_SOUTHEAST_1, + SmnRegion.AP_SOUTHEAST_3, + SmnRegion.CN_NORTH_2, + SmnRegion.CN_SOUTH_2, + SmnRegion.NA_MEXICO_1, + SmnRegion.LA_NORTH_2, + SmnRegion.SA_BRAZIL_1, + SmnRegion.LA_SOUTH_2, + SmnRegion.CN_NORTH_9 + ); + } +} diff --git a/nifi-huawei-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-huawei-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService new file mode 100644 index 0000000000000000000000000000000000000000..15ee89913a9761e2b94c75d7bf5e005f9fff1ae2 --- /dev/null +++ b/nifi-huawei-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.nifi.processors.huawei.credentials.provider.service.HuaweiCredentialsProviderControllerService + +org.apache.nifi.processors.huawei.obs.encryption.StandardOBSEncryptionService + diff --git a/nifi-huawei-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-huawei-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor new file mode 100644 index 0000000000000000000000000000000000000000..ea817f0952fb6d4b0c2295743b7fc5256f486df6 --- /dev/null +++ b/nifi-huawei-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.nifi.processors.huawei.obs.FetchOBSObject +org.apache.nifi.processors.huawei.obs.PutOBSObject +org.apache.nifi.processors.huawei.obs.DeleteOBSObject +org.apache.nifi.processors.huawei.obs.ListOBSObject +org.apache.nifi.processors.huawei.smn.PublishSMNMessage +org.apache.nifi.processors.huawei.dli.DLICreateSqlJob diff --git a/nifi-huawei-processors/src/main/resources/docs/org.apache.nifi.processors.aws.obs.encryption.StandardOBSEncryptionService/additionalDetails.html b/nifi-huawei-processors/src/main/resources/docs/org.apache.nifi.processors.aws.obs.encryption.StandardOBSEncryptionService/additionalDetails.html new file mode 100644 index 0000000000000000000000000000000000000000..881ca269f451e3ebc680a119e92d4e8e9aa453b8 --- /dev/null +++ b/nifi-huawei-processors/src/main/resources/docs/org.apache.nifi.processors.aws.obs.encryption.StandardOBSEncryptionService/additionalDetails.html @@ -0,0 +1,62 @@ + + + + + + ObsEncryptionService + + + + +

Description

+
+ The StandardOBSEncryptionService manages an encryption strategy and applies that strategy to various Obs operations. +
+
+ +

Configuration Details

+

Encryption Strategy

+ +
+ The name of the specific encryption strategy for this service to use when encrypting and decrypting S3 operations. + +
    +
  • None - no encryption is configured or applied.
    • +
  • Server-side KMS - encryption and decryption are performed by OBS using the configured KMS key.
    • +
  • Server-side Customer Key - encryption and decryption are performed by OBS using the supplied customer key.
    • + +
+
+ +

Key ID or Key Material

+

+ When configured for either the Server-side or Client-side KMS strategies, this field should contain the KMS Key ID. +

+

+ When configured for either the Server-side or Client-side Customer Key strategies, this field should contain the key + material, and that material must be base64 encoded. +

+

+ All other encryption strategies ignore this field. +

+ +

KMS Region

+
+ KMS key region, if any. This value must match the actual region of the KMS key if supplied. +
+ + + diff --git a/nifi-huawei-processors/src/main/resources/docs/org.apache.nifi.processors.huawei.obs.ListOBSObject/additionalDetails.html b/nifi-huawei-processors/src/main/resources/docs/org.apache.nifi.processors.huawei.obs.ListOBSObject/additionalDetails.html new file mode 100644 index 0000000000000000000000000000000000000000..290aec73384359897775fb6c086ea65959a4c1f3 --- /dev/null +++ b/nifi-huawei-processors/src/main/resources/docs/org.apache.nifi.processors.huawei.obs.ListOBSObject/additionalDetails.html @@ -0,0 +1,59 @@ + + + + + + + ListObs + + + + +

Streaming Versus Batch Processing

+ +

+ ListObs performs a listing of all Obs Objects that it encounters in the configured Obs bucket. + There are two common, broadly defined use cases. +

+ +

Streaming Use Case

+ +

+ By default, the Processor will create a separate FlowFile for each object in the bucket and add attributes for filename, bucket, etc. + A common use case is to connect ListOBS to the FetchOBS processor. These two processors used in conjunction with one another provide the ability to + easily monitor a bucket and fetch the contents of any new object as it lands in OBS in an efficient streaming fashion. +

+ +

Batch Use Case

+

+ Another common use case is the desire to process all newly arriving objects in a given bucket, and to then perform some action + only when all objects have completed their processing. The above approach of streaming the data makes this difficult, because NiFi is inherently + a streaming platform in that there is no "job" that has a beginning and an end. Data is simply picked up as it becomes available. +

+ +

+ To solve this, the ListOBS Processor can optionally be configured with a Record Writer. When a Record Writer is configured, a single + FlowFile will be created that will contain a Record for each object in the bucket, instead of a separate FlowFile per object. + See the documentation for ListFile for an example of how to build a dataflow that allows for processing all of the objects before proceeding + with any other step. +

+ +

+ One important difference between the data produced by ListFile and ListOBSObject, though, is the structure of the Records that are emitted. The Records + emitted by ListFile have a different schema than those emitted by ListOBS. +

+ + \ No newline at end of file diff --git a/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/AbstractOBSIT.java b/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/AbstractOBSIT.java new file mode 100644 index 0000000000000000000000000000000000000000..08535c015e8a4756708ea821dad2f7b6c506baf4 --- /dev/null +++ b/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/AbstractOBSIT.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.obs; + +import com.obs.services.ObsClient; +import com.obs.services.exception.ObsException; +import com.obs.services.model.*; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.processor.Processor; +import org.apache.nifi.processors.huawei.credentials.provider.service.HuaweiCredentialsProviderControllerService; +import org.apache.nifi.processors.huawei.obs.encryption.StandardOBSEncryptionService; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; +import static org.junit.Assert.fail; + +/** + * Base class for OBS Integration Tests. Establishes a bucket and helper methods for creating test scenarios + */ +public abstract class AbstractOBSIT { + + protected final static String ak = "your ak"; + protected final static String sk = "your sk"; + protected final static String SAMPLE_FILE_RESOURCE_NAME = "/hello.txt"; + protected final static String REGION_NAME = "cn-south-1"; + + // the endpoint of REGION_NAME + protected final static String ENDPOINT = "obs." + REGION_NAME +".myhuaweicloud.com"; + + // Static so multiple Tests can use same client, you may need to clean up uploaded files at the end of the method + protected static ObsClient client; + + protected static final String bucketName = "nifi-test" + System.currentTimeMillis() + "-" + REGION_NAME; + @BeforeAll + protected static void createClientAndBucket() { + // Creates a client and bucket for this test + try { + client = new ObsClient(ak, sk, OBSRegions.fromName(REGION_NAME).getEndpoint()); + if (client.headBucket(bucketName)) { + fail("Bucket " + bucketName + " exists. Choose a different bucket name to continue test"); + } + CreateBucketRequest request = new CreateBucketRequest(bucketName, REGION_NAME); + client.createBucket(request); + if (!client.headBucket(bucketName)) { + fail("Setup incomplete, tests will fail"); + } + } catch (final ObsException e) { + e.printStackTrace(); + fail("Can't create the key " + bucketName + ": " + e.getLocalizedMessage()); + } + } + + @AfterAll + public static void clear() { + // Empty the bucket before deleting it. + if (client == null) { + return; + } + try { + clearObjects(); + client.deleteBucket(bucketName); + } catch (final ObsException e) { + System.err.println("Unable to delete bucket " + bucketName + e); + } + if (client.headBucket(bucketName)) { + fail("Incomplete teardown, subsequent tests might fail"); + } + } + + protected static void clearObjects() { + if (client == null) { + return; + } + ListVersionsResult objectListing = client.listVersions(bucketName); + while (true) { + for (VersionOrDeleteMarker versionOrDeleteMarker : objectListing.getVersions()) { + client.deleteObject(bucketName, versionOrDeleteMarker.getObjectKey(), versionOrDeleteMarker.getVersionId()); + } + if (!objectListing.isTruncated()) { + break; + } + } + } + + protected static void clearObjects(String... keys) { + if (client == null || ArrayUtils.isEmpty(keys)) { + return; + } + List list = Arrays.asList(keys); + ObjectListing objectListing = client.listObjects(bucketName); + while (true) { + for (ObsObject objectSummary : objectListing.getObjects()) { + if (!list.contains(objectSummary.getObjectKey())) { + continue; + } + client.deleteObject(bucketName, objectSummary.getObjectKey()); + } + if (!objectListing.isTruncated()) { + break; + } + } + } + + protected synchronized void sleep() throws InterruptedException { + Thread.sleep(1000); + } + + protected void putTestFile(String... keys) throws ObsException, InterruptedException { + if (ArrayUtils.isEmpty(keys)) { + return; + } + File file = getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME); + + for (String key : keys) { + PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, file); + client.putObject(putRequest); + Thread.sleep(10); + } + } + + protected void putTestFileEncrypted(String key, File file) throws ObsException, FileNotFoundException { + PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, new FileInputStream(file)); + putRequest.setSseKmsHeader(new SseKmsHeader()); + client.putObject(putRequest); + } + + protected void putFileWithUserMetadata(String key, File file, Map userMetadata) throws ObsException, FileNotFoundException { + ObjectMetadata objectMetadata = new ObjectMetadata(); + objectMetadata.setUserMetadata(userMetadata); + PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, new FileInputStream(file)); + putRequest.setMetadata(objectMetadata); + client.putObject(putRequest); + } + + protected Path getResourcePath(String resourceName) { + Path path = null; + + try { + path = Paths.get(getClass().getResource(resourceName).toURI()); + } catch (URISyntaxException e) { + fail("Resource: " + resourceName + " does not exist" + e.getLocalizedMessage()); + } + + return path; + } + + protected File getFileFromResourceName(String resourceName) { + URI uri = null; + try { + uri = this.getClass().getResource(resourceName).toURI(); + } catch (URISyntaxException e) { + fail("Cannot proceed without File : " + resourceName); + } + + return new File(uri); + } + + protected static void addEncryptionService(TestRunner runner, AllowableValue allowableValue, String keyValue) { + ControllerService controllerService = new StandardOBSEncryptionService(); + try { + runner.addControllerService("standardObsEncryptionService", controllerService); + } catch (InitializationException e) { + throw new RuntimeException(e); + } + runner.setProperty(controllerService, StandardOBSEncryptionService.ENCRYPTION_STRATEGY, allowableValue); + runner.setProperty(controllerService, StandardOBSEncryptionService.ENCRYPTION_VALUE, keyValue); + runner.setProperty(controllerService, StandardOBSEncryptionService.KMS_REGION, REGION_NAME); + runner.enableControllerService(controllerService); + runner.assertValid(controllerService); + runner.setProperty(ENCRYPTION_SERVICE, "standardObsEncryptionService"); + } + + protected static TestRunner initRunner(Processor processor) { + final TestRunner runner = TestRunners.newTestRunner(processor); + runner.setProperty(OBS_REGION, REGION_NAME); + runner.setProperty(BUCKET, bucketName); + runner.setProperty(ACCESS_KEY, ak); + runner.setProperty(SECRET_KEY, sk); + return runner; + } + + protected static TestRunner initRunner(Processor processor, String endpoint) { + final TestRunner runner = TestRunners.newTestRunner(processor); + runner.setProperty(OBS_REGION, "NotFound"); + runner.setProperty(ENDPOINT_OVERRIDE_URL, endpoint); + runner.setProperty(BUCKET, bucketName); + runner.setProperty(ACCESS_KEY, ak); + runner.setProperty(SECRET_KEY, sk); + return runner; + } + + protected static TestRunner initCredentialsRunner(Processor processor) throws InitializationException { + final TestRunner runner = TestRunners.newTestRunner(processor); + runner.setProperty(OBS_REGION, REGION_NAME); + runner.setProperty(BUCKET, bucketName); + final HuaweiCredentialsProviderControllerService serviceImpl = new HuaweiCredentialsProviderControllerService(); + runner.addControllerService("huaweiCredentialsProvider", serviceImpl); + runner.setProperty(serviceImpl, ACCESS_KEY, ak); + runner.setProperty(serviceImpl, SECRET_KEY, sk); + runner.enableControllerService(serviceImpl); + runner.assertValid(serviceImpl); + runner.setProperty(HUAWEI_CREDENTIALS_PROVIDER_SERVICE, "huaweiCredentialsProvider"); + return runner; + } +} \ No newline at end of file diff --git a/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/ITDeleteOBSObject.java b/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/ITDeleteOBSObject.java new file mode 100644 index 0000000000000000000000000000000000000000..2d40b4e7f2da4bdd0704f4f977d21124492d160b --- /dev/null +++ b/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/ITDeleteOBSObject.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.obs; + +import org.apache.nifi.util.TestRunner; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.KEY; + + +/** + * Provides integration level testing with actual HuaweiYun OBS resources for {@link DeleteOBSObject} and requires additional configuration and resources to work. + */ +public class ITDeleteOBSObject extends AbstractOBSIT { + + @Test + public void testSimpleDelete() throws IOException, InterruptedException { + // Prepares for this test + putTestFile("delete-me"); + final TestRunner runner = initRunner(new DeleteOBSObject()); + final Map attrs = new HashMap<>(); + attrs.put("filename", "delete-me"); + runner.enqueue(new byte[0], attrs); + runner.run(1); + + runner.assertAllFlowFilesTransferred(DeleteOBSObject.REL_SUCCESS, 1); + } + + @Test + public void testDeleteFolder() throws IOException, InterruptedException { + // Prepares for this test + putTestFile("folder/delete-me"); + final TestRunner runner = initRunner(new DeleteOBSObject()); + final Map attrs = new HashMap<>(); + attrs.put("filename", "folder/delete-me"); + runner.enqueue(new byte[0], attrs); + runner.run(1); + + runner.assertAllFlowFilesTransferred(DeleteOBSObject.REL_SUCCESS, 1); + } + + @Test + public void testDeleteFolderUsingCredentialsProviderService() throws Throwable { + // Prepares for this test + putTestFile("folder/delete-me"); + final TestRunner runner = initCredentialsRunner(new DeleteOBSObject()); + + final Map attrs = new HashMap<>(); + attrs.put("filename", "folder/delete-me"); + runner.enqueue(new byte[0], attrs); + runner.run(1); + + runner.assertAllFlowFilesTransferred(DeleteOBSObject.REL_SUCCESS, 1); + } + + @Test + public void testDeleteFolderNoExpressionLanguage() throws IOException, InterruptedException { + // Prepares for this test + putTestFile("folder/delete-me"); + final TestRunner runner = initRunner(new DeleteOBSObject()); + runner.setProperty(KEY, "folder/delete-me"); + final Map attrs = new HashMap<>(); + attrs.put("filename", "a-different-name"); + runner.enqueue(new byte[0], attrs); + runner.run(1); + + runner.assertAllFlowFilesTransferred(DeleteOBSObject.REL_SUCCESS, 1); + } + + @Test + public void testTryToDeleteNotExistingFile() throws IOException { + final TestRunner runner = initRunner(new DeleteOBSObject()); + final Map attrs = new HashMap<>(); + attrs.put("filename", "no-such-a-file"); + runner.enqueue(new byte[0], attrs); + + runner.run(1); + + runner.assertAllFlowFilesTransferred(DeleteOBSObject.REL_SUCCESS, 1); + } + +} diff --git a/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/ITFetchOBSObject.java b/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/ITFetchOBSObject.java new file mode 100644 index 0000000000000000000000000000000000000000..ae2c80d08b58f86281d8fd075f20aac4ddcc7215 --- /dev/null +++ b/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/ITFetchOBSObject.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.obs; + +import com.obs.services.model.StorageClassEnum; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.nifi.processors.huawei.obs.FetchOBSObject.RANGE_LENGTH; +import static org.apache.nifi.processors.huawei.obs.FetchOBSObject.RANGE_START; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/** + * Provides integration level testing with actual HuaweiYun OBS resources for {@link FetchOBSObject} and requires additional configuration and resources to work. + */ +public class ITFetchOBSObject extends AbstractOBSIT { + @Test + public void testSimpleGet() throws IOException, InterruptedException { + putTestFile("test-file"); + final TestRunner runner = initRunner(new FetchOBSObject()); + final Map attrs = new HashMap<>(); + attrs.put("filename", "test-file"); + runner.enqueue(new byte[0], attrs); + + runner.run(1); + + runner.assertAllFlowFilesTransferred(FetchOBSObject.REL_SUCCESS, 1); + final List ffs = runner.getFlowFilesForRelationship(FetchOBSObject.REL_SUCCESS); + MockFlowFile ff = ffs.get(0); + ff.assertContentEquals(getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + } + + @Test + public void testSimpleGetEncrypted() throws IOException { + putTestFileEncrypted("test-file", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + final TestRunner runner = initRunner(new FetchOBSObject()); + final Map attrs = new HashMap<>(); + attrs.put("filename", "test-file"); + runner.enqueue(new byte[0], attrs); + + runner.run(1); + + runner.assertAllFlowFilesTransferred(FetchOBSObject.REL_SUCCESS, 1); + final List ffs = runner.getFlowFilesForRelationship(FetchOBSObject.REL_SUCCESS); + MockFlowFile ff = ffs.get(0); + ff.assertContentEquals(getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + } + + @Test + public void testTryToFetchNotExistingFile() throws IOException { + final TestRunner runner = initRunner(new FetchOBSObject()); + final Map attrs = new HashMap<>(); + attrs.put("filename", "no-such-a-file"); + runner.enqueue(new byte[0], attrs); + runner.run(1); + + runner.assertAllFlowFilesTransferred(FetchOBSObject.REL_FAILURE, 1); + } + + @Test + public void testContentsOfFileRetrieved() throws IOException, InterruptedException { + String key = "folder/1.txt"; + putTestFile(key); + final TestRunner runner = initRunner(new FetchOBSObject()); + final Map attrs = new HashMap<>(); + attrs.put("filename", key); + runner.enqueue(new byte[0], attrs); + runner.run(1); + + runner.assertAllFlowFilesTransferred(FetchOBSObject.REL_SUCCESS, 1); + final List ffs = runner.getFlowFilesForRelationship(FetchOBSObject.REL_SUCCESS); + final MockFlowFile out = ffs.iterator().next(); + final byte[] expectedBytes = Files.readAllBytes(getResourcePath(SAMPLE_FILE_RESOURCE_NAME)); + out.assertContentEquals(new String(expectedBytes)); + for (final Map.Entry entry : out.getAttributes().entrySet()) { + System.out.println(entry.getKey() + " : " + entry.getValue()); + } + } + + @Test + public void testFetchRangeOfFile() { + TestRunner putRunner = initRunner(new PutOBSObject()); + putRunner.setProperty(PutOBSObject.MULTIPART_THRESHOLD, "50 MB"); + putRunner.setProperty(PutOBSObject.MULTIPART_PART_SIZE, "50 MB"); + putRunner.setProperty(PutOBSObject.STORAGE_CLASS, StorageClassEnum.STANDARD.name()); + final Map putAttrs = new HashMap<>(); + String fileName = "testStorageClasses/large_" + StorageClassEnum.STANDARD + ".dat"; + putAttrs.put("filename", fileName); + putRunner.enqueue(new byte[50 * 1024 * 1024 + 1], putAttrs); + putRunner.run(); + putRunner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 1); + final TestRunner runner = initRunner(new FetchOBSObject()); + final Map attrs = new HashMap<>(); + attrs.put("filename", fileName); + runner.setProperty(RANGE_START,"1 MB"); + runner.setProperty(RANGE_LENGTH,"20 KB"); + runner.enqueue(new byte[0], attrs); + runner.run(1); + + runner.assertAllFlowFilesTransferred(FetchOBSObject.REL_SUCCESS, 1); + final List ffs = runner.getFlowFilesForRelationship(FetchOBSObject.REL_SUCCESS); + MockFlowFile ff = ffs.get(0); + long length = ff.getContent().length(); + assertEquals(length, 20480); + } + +} diff --git a/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/ITListOBSObject.java b/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/ITListOBSObject.java new file mode 100644 index 0000000000000000000000000000000000000000..e45d4bff26443147914a83c780c7322d1103258b --- /dev/null +++ b/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/ITListOBSObject.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.obs; + +import org.apache.nifi.processors.huawei.credentials.provider.service.HuaweiCredentialsProviderControllerService; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; + + +/** + * Provides integration level testing with actual Huawei OBS resources for {@link ListOBSObject} and requires additional configuration and resources to work. + * Because of the shared bucket, each test method stores objects into the same bucket, causing query confusion, so clearObjects is called after each method completes, + * and sleep is forced for a second before TestRunner is executed to avoid the effects between methods. + */ +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class ITListOBSObject extends AbstractOBSIT{ + @Test + public void testSimpleList() throws IOException, InterruptedException { + String[] putFiles = new String[]{"a","b/c","c/e","c/f","d/e"}; + putTestFile(putFiles); + final TestRunner runner = initRunner(new ListOBSObject()); + runner.setValidateExpressionUsage(false); + sleep(); + runner.run(); + runner.assertAllFlowFilesTransferred(ListOBSObject.REL_SUCCESS, 5); + sleep(); + // update + putTestFile(putFiles); + runner.run(); + + // repeat the cycle to ensure that all new objects are listed + // AbstractListProcessor.listByTrackingTimestamps + sleep(); + runner.run(); + runner.assertAllFlowFilesTransferred(ListOBSObject.REL_SUCCESS, 10); + List flowFiles = runner.getFlowFilesForRelationship(ListOBSObject.REL_SUCCESS); + flowFiles.get(0).assertAttributeEquals("filename", "a"); + flowFiles.get(1).assertAttributeEquals("filename", "b/c"); + flowFiles.get(2).assertAttributeEquals("filename", "c/e"); + flowFiles.get(3).assertAttributeEquals("filename", "c/f"); + flowFiles.get(4).assertAttributeEquals("filename", "d/e"); + clearObjects(putFiles); + } + + @Test + public void testRegionNotFound() throws IOException, InterruptedException { + String[] putFiles = new String[]{"a","b/c","c/e","c/f","d/e"}; + + putTestFile(putFiles); + final TestRunner runner = initRunner(new ListOBSObject(), ENDPOINT); + runner.setValidateExpressionUsage(false); + sleep(); + runner.run(); + runner.assertAllFlowFilesTransferred(ListOBSObject.REL_SUCCESS, 5); + sleep(); + // update + putTestFile(putFiles); + runner.run(); + + // repeat the cycle to ensure that all new objects are listed + // AbstractListProcessor.listByTrackingTimestamps + sleep(); + runner.run(); + runner.assertAllFlowFilesTransferred(ListOBSObject.REL_SUCCESS, 10); + List flowFiles = runner.getFlowFilesForRelationship(ListOBSObject.REL_SUCCESS); + flowFiles.get(0).assertAttributeEquals("filename", "a"); + flowFiles.get(1).assertAttributeEquals("filename", "b/c"); + flowFiles.get(2).assertAttributeEquals("filename", "c/e"); + flowFiles.get(3).assertAttributeEquals("filename", "c/f"); + flowFiles.get(4).assertAttributeEquals("filename", "d/e"); + clearObjects(putFiles); + } + + @Test + public void testSimpleListUsingCredentialsProviderService() throws Throwable { + String[] putFiles = new String[]{"a","b/c","d/e"}; + putTestFile(putFiles); + final TestRunner runner = initRunner(new ListOBSObject()); + final HuaweiCredentialsProviderControllerService serviceImpl = new HuaweiCredentialsProviderControllerService(); + runner.addControllerService("huaweiCredentialsProvider", serviceImpl); + runner.setProperty(serviceImpl, ACCESS_KEY, ak); + runner.setProperty(serviceImpl, SECRET_KEY, sk); + runner.enableControllerService(serviceImpl); + runner.assertValid(serviceImpl); + runner.setProperty(HUAWEI_CREDENTIALS_PROVIDER_SERVICE, "huaweiCredentialsProvider"); + runner.setValidateExpressionUsage(false); + runner.run(); + // repeat the cycle to ensure that all new objects are listed + // AbstractListProcessor.listByTrackingTimestamps + sleep(); + runner.run(); + runner.assertAllFlowFilesTransferred(ListOBSObject.REL_SUCCESS, 3); + List flowFiles = runner.getFlowFilesForRelationship(ListOBSObject.REL_SUCCESS); + flowFiles.get(0).assertAttributeEquals("filename", "a"); + flowFiles.get(1).assertAttributeEquals("filename", "b/c"); + flowFiles.get(2).assertAttributeEquals("filename", "d/e"); + clearObjects(putFiles); + } + + @Test + public void testSimpleListWithDelimiter() throws Throwable { + String[] putFiles = new String[]{"dm1","dm2/c","dm2_/e"}; + putTestFile(putFiles); + final TestRunner runner = initRunner(new ListOBSObject()); + runner.setProperty(DELIMITER, "2"); + sleep(); + runner.run(); + + runner.assertAllFlowFilesTransferred(ListOBSObject.REL_SUCCESS, 1); + List flowFiles = runner.getFlowFilesForRelationship(ListOBSObject.REL_SUCCESS); + flowFiles.get(0).assertAttributeEquals("filename", "dm1"); + clearObjects(putFiles); + } + + @Test + public void testSimpleListWithPrefixAndDelimiter() throws Throwable { + String[] putFiles = new String[]{"tdm1","tdm2/c","tdm2_/e"}; + putTestFile(putFiles); + final TestRunner runner = initRunner(new ListOBSObject()); + runner.setProperty(PREFIX, "t"); + runner.setProperty(DELIMITER, "1"); + sleep(); + runner.run(); + + runner.assertAllFlowFilesTransferred(ListOBSObject.REL_SUCCESS, 2); + List flowFiles = runner.getFlowFilesForRelationship(ListOBSObject.REL_SUCCESS); + flowFiles.get(0).assertAttributeEquals("filename", "tdm2/c"); + clearObjects(putFiles); + } + + @Test + public void testSimpleListWithPrefix() throws Throwable { + String[] putFiles = new String[]{"a","bc","b/c","d/e"}; + putTestFile(putFiles); + final TestRunner runner = initRunner(new ListOBSObject()); + runner.setProperty(PREFIX, "b"); + sleep(); + runner.run(); + + runner.assertAllFlowFilesTransferred(ListOBSObject.REL_SUCCESS, 2); + List flowFiles = runner.getFlowFilesForRelationship(ListOBSObject.REL_SUCCESS); + flowFiles.get(0).assertAttributeEquals("filename", "bc"); + flowFiles.get(1).assertAttributeEquals("filename", "b/c"); + clearObjects(putFiles); + } + + @Test + public void testUserMetadataWritten() throws FileNotFoundException, InterruptedException { + String key = "b/fileWithUserMetadata"; + Map userMetadata = new HashMap<>(); + userMetadata.put("dummy.metadata.1", "dummyvalue1"); + userMetadata.put("dummy.metadata.2", "dummyvalue2"); + putFileWithUserMetadata(key, getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME), userMetadata); + final TestRunner runner = initRunner(new ListOBSObject()); + runner.setProperty(PREFIX, "b/"); + runner.setProperty(WRITE_USER_METADATA, "true"); + sleep(); + runner.run(); + + runner.assertAllFlowFilesTransferred(ListOBSObject.REL_SUCCESS, 1); + + MockFlowFile flowFiles = runner.getFlowFilesForRelationship(ListOBSObject.REL_SUCCESS).get(0); + flowFiles.assertAttributeEquals("filename", "b/fileWithUserMetadata"); + flowFiles.assertAttributeExists("obs.userMetadata.dummy.metadata.1"); + flowFiles.assertAttributeExists("obs.userMetadata.dummy.metadata.2"); + flowFiles.assertAttributeEquals("obs.userMetadata.dummy.metadata.1", "dummyvalue1"); + flowFiles.assertAttributeEquals("obs.userMetadata.dummy.metadata.2", "dummyvalue2"); + clearObjects(key); + } +} diff --git a/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/ITPutOBSObject.java b/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/ITPutOBSObject.java new file mode 100644 index 0000000000000000000000000000000000000000..79839e77a496c70d4889673023d60cb9aa4d895f --- /dev/null +++ b/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/obs/ITPutOBSObject.java @@ -0,0 +1,902 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.obs; + +import com.obs.services.ObsClient; +import com.obs.services.exception.ObsException; +import com.obs.services.model.*; +import org.apache.commons.codec.binary.Base64; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.processor.DataUnit; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processors.huawei.obs.encryption.StandardOBSEncryptionService; +import org.apache.nifi.provenance.ProvenanceEventRecord; +import org.apache.nifi.provenance.ProvenanceEventType; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.SecureRandom; +import java.util.*; + +import static org.apache.nifi.processors.huawei.obs.Constants.*; +import static org.apache.nifi.processors.huawei.common.PropertyDescriptors.*; +import static org.junit.jupiter.api.Assertions.*; + +/** + * Provides integration level testing with actual Huawei OBS resources for {@link PutOBSObject} and requires additional configuration and resources to work. + */ +public class ITPutOBSObject extends AbstractOBSIT { + + final static String TEST_PARTSIZE_STRING = "50 mb"; + final static Long TEST_PARTSIZE_LONG = 50L * 1024L * 1024L; + + final static Long S3_MINIMUM_PART_SIZE = 50L * 1024L * 1024L; + final static Long S3_MAXIMUM_OBJECT_SIZE = 5L * 1024L * 1024L * 1024L; + + private static String kmsKeyId = "your kmsId in this region"; + + private static String randomKeyMaterial = ""; + + private static String USER_ID = "**The account ids of the authorized user, separated by commas (,)**"; + private static String OWNER_ID = "**Account ID of Owner**"; + + + @BeforeAll + public static void setupClass() { + byte[] keyRawBytes = new byte[32]; + SecureRandom secureRandom = new SecureRandom(); + secureRandom.nextBytes(keyRawBytes); + randomKeyMaterial = Base64.encodeBase64String(keyRawBytes); + System.out.println(randomKeyMaterial); + } + + @Test + public void testSimplePut() throws IOException { + TestRunner runner = initRunner(new PutOBSObject()); + assertTrue(runner.setProperty("x-custom-prop", "hello").isValid()); + for (int i = 0; i < 3; i++) { + final Map attrs = new HashMap<>(); + attrs.put("filename", i + ".txt"); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + } + runner.run(3); + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 3); + } + + @Test + public void testSimplePutEncrypted() throws IOException { + TestRunner runner = initRunner(new PutOBSObject()); + addEncryptionService(runner, StandardOBSEncryptionService.SSE_KMS, kmsKeyId); + + for (int i = 0; i < 3; i++) { + final Map attrs = new HashMap<>(); + attrs.put("filename", i + ".txt"); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + } + runner.run(3); + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 3); + final List ffs = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + for (MockFlowFile flowFile : ffs) { + flowFile.assertAttributeEquals(OBS_ENCRYPTION_STRATEGY, StandardOBSEncryptionService.SSE_KMS.getValue()); + } + } + + @Test + public void testSimplePutFilenameWithNationalCharacters() throws IOException { + TestRunner runner = initRunner(new PutOBSObject()); + + final Map attrs = new HashMap<>(); + attrs.put("filename", "测试文件名.txt"); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + + runner.run(1); + + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 1); + } + + // put & fetch & compare + private void testPutThenFetch(String encryptionStrategy) throws IOException { + // Put + TestRunner putRunner = initRunner(new PutOBSObject()); + + // Server-side KMS + if(StandardOBSEncryptionService.SSE_KMS.getValue().equals(encryptionStrategy)){ + addEncryptionService(putRunner, StandardOBSEncryptionService.SSE_KMS, kmsKeyId); + } else if (StandardOBSEncryptionService.SSE_C.getValue().equals(encryptionStrategy)){ + // Server-side Customer Key + addEncryptionService(putRunner, StandardOBSEncryptionService.SSE_C, randomKeyMaterial); + } + + final Map attrs = new HashMap<>(); + attrs.put("filename", "filename-obs.txt"); + putRunner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + putRunner.run(); + + putRunner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 1); + List ffs = putRunner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + ffs.get(0).assertAttributeEquals(OBS_ENCRYPTION_STRATEGY, encryptionStrategy); + + // Fetch + TestRunner fetchRunner = initRunner(new FetchOBSObject()); + // Server-side KMS + if(StandardOBSEncryptionService.SSE_KMS.getValue().equals(encryptionStrategy)){ + addEncryptionService(fetchRunner, StandardOBSEncryptionService.SSE_KMS, kmsKeyId); + } else if (StandardOBSEncryptionService.SSE_C.getValue().equals(encryptionStrategy)){ + // Server-side Customer Key + addEncryptionService(fetchRunner, StandardOBSEncryptionService.SSE_C, randomKeyMaterial); + } + fetchRunner.enqueue(new byte[0], attrs); + fetchRunner.run(1); + + fetchRunner.assertAllFlowFilesTransferred(FetchOBSObject.REL_SUCCESS, 1); + ffs = fetchRunner.getFlowFilesForRelationship(FetchOBSObject.REL_SUCCESS); + MockFlowFile ff = ffs.get(0); + // 比较文件内容是否一致 + ff.assertContentEquals(getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + ff.assertAttributeEquals(OBS_ENCRYPTION_STRATEGY, StandardOBSEncryptionService.STRATEGY_NAME_SSE_C.equals(encryptionStrategy) ? encryptionStrategy : null); + } + + @Test + public void testPutThenFetchWithNONE() throws IOException { + testPutThenFetch(StandardOBSEncryptionService.STRATEGY_NAME_NONE); + } + + @Test + public void testPutThenFetchWithSSE() throws IOException { + testPutThenFetch(StandardOBSEncryptionService.STRATEGY_NAME_SSE_KMS); + } + + @Test + public void testPutThenFetchWithSSEC() throws IOException { + testPutThenFetch(StandardOBSEncryptionService.STRATEGY_NAME_SSE_C); + } + + + @Test + public void testPutObsObjectUsingCredentialsProviderService() throws Throwable { + final TestRunner runner = initCredentialsRunner(new PutOBSObject()); + for (int i = 0; i < 3; i++) { + final Map attrs = new HashMap<>(); + attrs.put("filename", i + ".txt"); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + } + runner.run(3); + + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 3); + + } + + @Test + public void testMetaData() throws IOException { + PutOBSObject processor = new PutOBSObject(); + final TestRunner runner = initRunner(new PutOBSObject()); + PropertyDescriptor prop1 = processor.getSupportedDynamicPropertyDescriptor("TEST-PROP-1"); + runner.setProperty(prop1, "TESTING-1-2-3"); + PropertyDescriptor prop2 = processor.getSupportedDynamicPropertyDescriptor("TEST-PROP-2"); + runner.setProperty(prop2, "TESTING-4-5-6"); + + final Map attrs = new HashMap<>(); + attrs.put("filename", "meta.txt"); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + + runner.run(); + + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 1); + List flowFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + MockFlowFile ff1 = flowFiles.get(0); + for (Map.Entry attrib : ff1.getAttributes().entrySet()) { + System.out.println(attrib.getKey() + " = " + attrib.getValue()); + } + } + + @Test + public void testContentType() throws IOException { + TestRunner runner = initRunner(new PutOBSObject()); + runner.setProperty(PutOBSObject.CONTENT_TYPE, "text/plain"); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME)); + runner.run(); + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 1); + List flowFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + MockFlowFile ff1 = flowFiles.get(0); + ff1.assertAttributeEquals(OBS_CONTENT_TYPE, "text/plain"); + } + + @Test + public void testContentDispositionInline() throws IOException { + TestRunner runner = initRunner(new PutOBSObject()); + runner.setProperty(PutOBSObject.CONTENT_DISPOSITION, PutOBSObject.CONTENT_DISPOSITION_INLINE); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME)); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 1); + List flowFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + MockFlowFile ff1 = flowFiles.get(0); + ff1.assertAttributeEquals(OBS_CONTENT_DISPOSITION, PutOBSObject.CONTENT_DISPOSITION_INLINE); + } + + // ok + @Test + public void testContentDispositionNull() throws IOException { + // Put + TestRunner runner =initRunner(new PutOBSObject()); + + final Map attrs = new HashMap<>(); + attrs.put("filename", "filename-on-s3.txt"); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 1); + List ffs = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + + // Fetch + TestRunner fetchRunner = initRunner(new FetchOBSObject()); + fetchRunner.enqueue(new byte[0], attrs); + fetchRunner.run(1); + + fetchRunner.assertAllFlowFilesTransferred(FetchOBSObject.REL_SUCCESS, 1); + ffs = fetchRunner.getFlowFilesForRelationship(FetchOBSObject.REL_SUCCESS); + MockFlowFile ff = ffs.get(0); + ff.assertContentEquals(getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME)); + ff.assertAttributeNotExists(OBS_CONTENT_DISPOSITION); + } + + + //ok + @Test + public void testContentDispositionAttachment() throws IOException { + TestRunner runner = initRunner(new PutOBSObject()); + runner.setProperty(PutOBSObject.CONTENT_DISPOSITION, PutOBSObject.CONTENT_DISPOSITION_ATTACHMENT); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME)); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 1); + List flowFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + MockFlowFile ff1 = flowFiles.get(0); + ff1.assertAttributeEquals(OBS_CONTENT_DISPOSITION, "attachment; filename=\"hello.txt\""); + } + + @Test + public void testCacheControl() throws IOException { + TestRunner runner = initRunner(new PutOBSObject()); + + runner.setProperty(PutOBSObject.CACHE_CONTROL, "no-cache"); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME)); + + runner.run(); + + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 1); + List flowFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + MockFlowFile ff1 = flowFiles.get(0); + ff1.assertAttributeEquals(OBS_CACHE_CONTROL, "no-cache"); + } + + @Test + public void testPutInFolder() throws IOException { + TestRunner runner = initRunner(new PutOBSObject()); + assertTrue(runner.setProperty("x-custom-prop", "hello").isValid()); + runner.assertValid(); + final Map attrs = new HashMap<>(); + attrs.put("filename", "folder/1.txt"); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + runner.run(); + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 1); + } + + // ok + @Test + public void testStorageClasses() throws IOException { + TestRunner runner = initRunner(new PutOBSObject()); + + assertTrue(runner.setProperty("x-custom-prop", "hello").isValid()); + + for (StorageClassEnum storageClass : StorageClassEnum.values()) { + + runner.setProperty(PutOBSObject.STORAGE_CLASS, storageClass.name()); + + final Map attrs = new HashMap<>(); + attrs.put("filename", "testStorageClasses/small_" + storageClass.name() + ".txt"); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + + runner.run(); + + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 1); + FlowFile file = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS).get(0); + assertEquals(storageClass.toString(), file.getAttribute(OBS_STORAGE_CLASS)); + + runner.clearTransferState(); + } + } + + // ok + @Test + public void testStorageClassesMultipart() throws IOException { + TestRunner runner = initRunner(new PutOBSObject()); + runner.setProperty(PutOBSObject.MULTIPART_THRESHOLD, "50 MB"); + runner.setProperty(PutOBSObject.MULTIPART_PART_SIZE, "50 MB"); + assertTrue(runner.setProperty("x-custom-prop", "hello").isValid()); + + for (StorageClassEnum storageClass : StorageClassEnum.values()) { + runner.setProperty(PutOBSObject.STORAGE_CLASS, storageClass.name()); + final Map attrs = new HashMap<>(); + attrs.put("filename", "testStorageClasses/large_" + storageClass.name() + ".dat"); + runner.enqueue(new byte[50 * 1024 * 1024 + 1], attrs); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 1); + FlowFile file = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS).get(0); + assertEquals(storageClass.toString(), file.getAttribute(OBS_STORAGE_CLASS)); + runner.clearTransferState(); + } + } + + @Test + public void testPermissions() throws IOException { + TestRunner runner = initRunner(new PutOBSObject()); + + runner.setProperty(OWNER, OWNER_ID); + runner.setProperty(READ_USER_LIST, USER_ID); + runner.setProperty(WRITE_ACL_LIST, USER_ID); + runner.setProperty(READ_ACL_LIST, USER_ID); + final Map attrs = new HashMap<>(); + attrs.put("filename", "4.txt"); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + + runner.run(); + + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS, 1); + } + + @Test + public void testDynamicProperty() throws IOException { + final String DYNAMIC_ATTRIB_KEY = "fs.runTimestamp"; + final String DYNAMIC_ATTRIB_VALUE = "${now():toNumber()}"; + + + final TestRunner runner = initRunner(new PutOBSObject()); + final PutOBSObject processor = (PutOBSObject)runner.getProcessor(); + runner.setProperty(PutOBSObject.MULTIPART_PART_SIZE, TEST_PARTSIZE_STRING); + PropertyDescriptor testAttrib = processor.getSupportedDynamicPropertyDescriptor(DYNAMIC_ATTRIB_KEY); + runner.setProperty(testAttrib, DYNAMIC_ATTRIB_VALUE); + + final String FILE1_NAME = "file1"; + Map attribs = new HashMap<>(); + attribs.put(CoreAttributes.FILENAME.key(), FILE1_NAME); + runner.enqueue("123".getBytes(), attribs); + + runner.assertValid(); + processor.getPropertyDescriptor(DYNAMIC_ATTRIB_KEY); + runner.run(); + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS); + final List successFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + assertEquals(1, successFiles.size()); + MockFlowFile ff1 = successFiles.get(0); + + Long now = System.currentTimeMillis(); + String millisNow = Long.toString(now); + String millisOneSecAgo = Long.toString(now - 1000L); + String usermeta = ff1.getAttribute(OBS_USER_META); + String[] usermetaLine0 = usermeta.split(System.lineSeparator())[0].split("="); + String usermetaKey0 = usermetaLine0[0]; + String usermetaValue0 = usermetaLine0[1]; + assertEquals(DYNAMIC_ATTRIB_KEY, usermetaKey0); + assertTrue(usermetaValue0.compareTo(millisOneSecAgo) >=0 && usermetaValue0.compareTo(millisNow) <= 0); + } + + @Test + public void testProvenance() throws InitializationException { + final String PROV1_FILE = "provfile1"; + + final TestRunner runner = initRunner(new PutOBSObject()); + + runner.setProperty(KEY, "${filename}"); + + Map attribs = new HashMap<>(); + attribs.put(CoreAttributes.FILENAME.key(), PROV1_FILE); + runner.enqueue("prov1 contents".getBytes(), attribs); + + runner.assertValid(); + runner.run(); + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS); + final List successFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + assertEquals(1, successFiles.size()); + + final List provenanceEvents = runner.getProvenanceEvents(); + assertEquals(1, provenanceEvents.size()); + ProvenanceEventRecord provRec1 = provenanceEvents.get(0); + assertEquals(ProvenanceEventType.SEND, provRec1.getEventType()); + assertEquals(runner.getProcessor().getIdentifier(), provRec1.getComponentId()); + assertEquals(10, provRec1.getUpdatedAttributes().size()); + assertEquals(bucketName, provRec1.getUpdatedAttributes().get(OBS_BUCKET)); + } + + @Test + public void testStateDefaults() { + PutOBSObject.MultipartState state1 = new PutOBSObject.MultipartState(); + assertEquals(state1.getUploadId(), ""); + assertEquals(state1.getFilePosition(), (Long) 0L); + assertEquals(state1.getPartETags().size(), 0L); + assertEquals(state1.getPartSize(), (Long) 0L); + assertEquals(state1.getStorageClass().toString(), StorageClassEnum.STANDARD.toString()); + assertEquals(state1.getContentLength(), (Long) 0L); + } + + @Test + public void testMultipartProperties() throws IOException { + final TestRunner runner = initRunner(new PutOBSObject()); + final ProcessContext context = runner.getProcessContext(); + runner.setProperty(FULL_CONTROL_USER_LIST, USER_ID); + runner.setProperty(PutOBSObject.MULTIPART_PART_SIZE, TEST_PARTSIZE_STRING); + + runner.setProperty(KEY, AbstractOBSIT.SAMPLE_FILE_RESOURCE_NAME); + + assertEquals(bucketName, context.getProperty(BUCKET).toString()); + assertEquals(SAMPLE_FILE_RESOURCE_NAME, context.getProperty(KEY).getValue()); + assertEquals(TEST_PARTSIZE_LONG.longValue(), context.getProperty(PutOBSObject.MULTIPART_PART_SIZE).asDataSize(DataUnit.B).longValue()); + } + + @Test + public void testLocalStatePersistence() throws IOException { + final PutOBSObject processor = new PutOBSObject(); + final TestRunner runner = TestRunners.newTestRunner(processor); + runner.setValidateExpressionUsage(false); + final String bucket = runner.getProcessContext().getProperty(BUCKET).getValue(); + final String key = runner.getProcessContext().getProperty(KEY).getValue(); + final String cacheKey1 = runner.getProcessor().getIdentifier() + "/" + bucket + "/" + key; + final String cacheKey2 = runner.getProcessor().getIdentifier() + "/" + bucket + "/" + key + "-v2"; + final String cacheKey3 = runner.getProcessor().getIdentifier() + "/" + bucket + "/" + key + "-v3"; + + /* + * store 3 versions of state + */ + PutOBSObject.MultipartState state1orig = new PutOBSObject.MultipartState(); + processor.persistLocalState(cacheKey1, state1orig); + + PutOBSObject.MultipartState state2orig = new PutOBSObject.MultipartState(); + state2orig.setUploadId("1234"); + state2orig.setContentLength(1234L); + processor.persistLocalState(cacheKey2, state2orig); + + PutOBSObject.MultipartState state3orig = new PutOBSObject.MultipartState(); + state3orig.setUploadId("5678"); + state3orig.setContentLength(5678L); + processor.persistLocalState(cacheKey3, state3orig); + + final List uploadList = new ArrayList<>(); + final MultipartUpload upload1 = new MultipartUpload("", key,new Date(), StorageClassEnum.STANDARD, null, null); + uploadList.add(upload1); + final MultipartUpload upload2 = new MultipartUpload("1234", key + "-v2",new Date(), StorageClassEnum.STANDARD, null, null); + uploadList.add(upload2); + final MultipartUpload upload3 = new MultipartUpload("5678", key + "-v3",new Date(), StorageClassEnum.STANDARD, null, null); + uploadList.add(upload3); + final MultipartUploadListing uploadListing = new MultipartUploadListing.Builder().multipartTaskList(uploadList).builder(); + final MockAmazonS3Client mockClient = new MockAmazonS3Client(OBSRegions.fromName(REGION_NAME).getEndpoint()); + mockClient.setListing(uploadListing); + + /* + * reload and validate stored state + */ + final PutOBSObject.MultipartState state1new = processor.getLocalStateIfInOBS(mockClient, bucket, cacheKey1); + assertEquals("", state1new.getUploadId()); + assertEquals(0L, state1new.getFilePosition().longValue()); + assertEquals(new ArrayList(), state1new.getPartETags()); + assertEquals(0L, state1new.getPartSize().longValue()); + assertEquals(StorageClassEnum.STANDARD.toString(), state1new.getStorageClass().getCode()); + assertEquals(0L, state1new.getContentLength().longValue()); + + final PutOBSObject.MultipartState state2new = processor.getLocalStateIfInOBS(mockClient, bucket, cacheKey2); + assertEquals("1234", state2new.getUploadId()); + assertEquals(0L, state2new.getFilePosition().longValue()); + assertEquals(new ArrayList(), state2new.getPartETags()); + assertEquals(0L, state2new.getPartSize().longValue()); + assertEquals(StorageClassEnum.STANDARD.toString(), state2new.getStorageClass().getCode()); + assertEquals(1234L, state2new.getContentLength().longValue()); + + final PutOBSObject.MultipartState state3new = processor.getLocalStateIfInOBS(mockClient, bucket, cacheKey3); + assertEquals("5678", state3new.getUploadId()); + assertEquals(0L, state3new.getFilePosition().longValue()); + assertEquals(new ArrayList(), state3new.getPartETags()); + assertEquals(0L, state3new.getPartSize().longValue()); + assertEquals(StorageClassEnum.STANDARD.toString(), state3new.getStorageClass().getCode()); + assertEquals(5678L, state3new.getContentLength().longValue()); + } + + @Test + public void testStatePersistsETags() throws IOException { + final PutOBSObject processor = new PutOBSObject(); + final TestRunner runner = TestRunners.newTestRunner(processor); + runner.setValidateExpressionUsage(false); + final String bucket = runner.getProcessContext().getProperty(BUCKET).getValue(); + final String key = runner.getProcessContext().getProperty(KEY).getValue(); + final String cacheKey1 = runner.getProcessor().getIdentifier() + "/" + bucket + "/" + key + "-bv1"; + final String cacheKey2 = runner.getProcessor().getIdentifier() + "/" + bucket + "/" + key + "-bv2"; + final String cacheKey3 = runner.getProcessor().getIdentifier() + "/" + bucket + "/" + key + "-bv3"; + + /* + * store 3 versions of state + */ + PutOBSObject.MultipartState state1orig = new PutOBSObject.MultipartState(); + processor.persistLocalState(cacheKey1, state1orig); + + PutOBSObject.MultipartState state2orig = new PutOBSObject.MultipartState(); + state2orig.setUploadId("1234"); + state2orig.setContentLength(1234L); + processor.persistLocalState(cacheKey2, state2orig); + + PutOBSObject.MultipartState state3orig = new PutOBSObject.MultipartState(); + state3orig.setUploadId("5678"); + state3orig.setContentLength(5678L); + processor.persistLocalState(cacheKey3, state3orig); + + /* + * persist state to caches so that + * 1. v2 has 2 and then 4 tags + * 2. v3 has 4 and then 2 tags + */ + state2orig.getPartETags().add(new PartEtag("state 2 tag one", 1)); + state2orig.getPartETags().add(new PartEtag("state 2 tag two", 2)); + processor.persistLocalState(cacheKey2, state2orig); + state2orig.getPartETags().add(new PartEtag("state 2 tag three", 3)); + state2orig.getPartETags().add(new PartEtag("state 2 tag four", 4)); + processor.persistLocalState(cacheKey2, state2orig); + + state3orig.getPartETags().add(new PartEtag("state 3 tag one", 1)); + state3orig.getPartETags().add(new PartEtag("state 3 tag two", 2)); + state3orig.getPartETags().add(new PartEtag("state 3 tag three", 3)); + state3orig.getPartETags().add(new PartEtag("state 3 tag four", 4)); + processor.persistLocalState(cacheKey3, state3orig); + state3orig.getPartETags().remove(state3orig.getPartETags().size() - 1); + state3orig.getPartETags().remove(state3orig.getPartETags().size() - 1); + processor.persistLocalState(cacheKey3, state3orig); + + final List uploadList = new ArrayList<>(); + final MultipartUpload upload1 = new MultipartUpload("1234", key + "-bv2",new Date(), StorageClassEnum.STANDARD, null, null); + uploadList.add(upload1); + final MultipartUpload upload2 = new MultipartUpload("5678", key + "-bv3",new Date(), StorageClassEnum.STANDARD, null, null); + uploadList.add(upload2); + final MultipartUploadListing uploadListing = new MultipartUploadListing.Builder().multipartTaskList(uploadList).builder(); + final MockAmazonS3Client mockClient = new MockAmazonS3Client(OBSRegions.fromName(REGION_NAME).getEndpoint()); + mockClient.setListing(uploadListing); + + /* + * load state and validate that + * 1. v2 restore shows 4 tags + * 2. v3 restore shows 2 tags + */ + final PutOBSObject.MultipartState state2new = processor.getLocalStateIfInOBS(mockClient, bucket, cacheKey2); + assertEquals("1234", state2new.getUploadId()); + assertEquals(4, state2new.getPartETags().size()); + + final PutOBSObject.MultipartState state3new = processor.getLocalStateIfInOBS(mockClient, bucket, cacheKey3); + assertEquals("5678", state3new.getUploadId()); + assertEquals(2, state3new.getPartETags().size()); + } + + @Test + public void testStateRemove() throws IOException { + final PutOBSObject processor = new PutOBSObject(); + final TestRunner runner = TestRunners.newTestRunner(processor); + runner.setValidateExpressionUsage(false); + final String bucket = runner.getProcessContext().getProperty(BUCKET).getValue(); + final String key = runner.getProcessContext().getProperty(KEY).getValue(); + final String cacheKey = runner.getProcessor().getIdentifier() + "/" + bucket + "/" + key + "-sr"; + + final List uploadList = new ArrayList<>(); + final MultipartUpload upload1 = new MultipartUpload("1234", key ,new Date(), StorageClassEnum.STANDARD, null, null); + uploadList.add(upload1); + final MultipartUploadListing uploadListing = new MultipartUploadListing.Builder().multipartTaskList(uploadList).builder(); + final MockAmazonS3Client mockClient = new MockAmazonS3Client(OBSRegions.fromName(REGION_NAME).getEndpoint()); + mockClient.setListing(uploadListing); + + /* + * store state, retrieve and validate, remove and validate + */ + PutOBSObject.MultipartState stateOrig = new PutOBSObject.MultipartState(); + stateOrig.setUploadId("1234"); + stateOrig.setContentLength(1234L); + processor.persistLocalState(cacheKey, stateOrig); + + PutOBSObject.MultipartState state1 = processor.getLocalStateIfInOBS(mockClient, bucket, cacheKey); + assertEquals("1234", state1.getUploadId()); + assertEquals(1234L, state1.getContentLength().longValue()); + + processor.persistLocalState(cacheKey, null); + PutOBSObject.MultipartState state2 = processor.getLocalStateIfInOBS(mockClient, bucket, cacheKey); + assertNull(state2); + } + + @Test + public void testMultipartSmallerThanMinimum() throws IOException { + final String FILE1_NAME = "file1"; + + final byte[] megabyte = new byte[1024 * 1024]; + final Path tempFile = Files.createTempFile("s3mulitpart", "tmp"); + final FileOutputStream tempOut = new FileOutputStream(tempFile.toFile()); + long tempByteCount = 0; + for (int i = 0; i < 5; i++) { + tempOut.write(megabyte); + tempByteCount += megabyte.length; + } + tempOut.close(); + System.out.println("file size: " + tempByteCount); + assertTrue(tempByteCount < S3_MINIMUM_PART_SIZE); + + assertTrue(megabyte.length < S3_MINIMUM_PART_SIZE); + assertTrue(TEST_PARTSIZE_LONG >= S3_MINIMUM_PART_SIZE && TEST_PARTSIZE_LONG <= S3_MAXIMUM_OBJECT_SIZE); + + final TestRunner runner = initRunner(new PutOBSObject()); + runner.setProperty(PutOBSObject.MULTIPART_PART_SIZE, TEST_PARTSIZE_STRING); + + Map attributes = new HashMap<>(); + attributes.put(CoreAttributes.FILENAME.key(), FILE1_NAME); + runner.enqueue(new FileInputStream(tempFile.toFile()), attributes); + + runner.assertValid(); + runner.run(); + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS); + final List successFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + assertEquals(1, successFiles.size()); + final List failureFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_FAILURE); + assertEquals(0, failureFiles.size()); + MockFlowFile ff1 = successFiles.get(0); + assertEquals(OBS_API_METHOD_PUT_OBJECT, ff1.getAttribute(OBS_API_METHOD_ATTR_KEY)); + assertEquals(FILE1_NAME, ff1.getAttribute(CoreAttributes.FILENAME.key())); + assertEquals(bucketName, ff1.getAttribute(OBS_BUCKET)); + assertEquals(FILE1_NAME, ff1.getAttribute(OBS_OBJECT)); + assertEquals(tempByteCount, ff1.getSize()); + } + + @Test + public void testMultipartBetweenMinimumAndMaximum() throws IOException { + final String FILE1_NAME = "file1"; + + final byte[] megabyte = new byte[1024 * 1024]; + final Path tempFile = Files.createTempFile("s3mulitpart", "tmp"); + final FileOutputStream tempOut = new FileOutputStream(tempFile.toFile()); + long tempByteCount = 0; + for ( ; tempByteCount < TEST_PARTSIZE_LONG + 1; ) { + tempOut.write(megabyte); + tempByteCount += megabyte.length; + } + tempOut.close(); + System.out.println("file size: " + tempByteCount); + assertTrue(tempByteCount > S3_MINIMUM_PART_SIZE && tempByteCount < S3_MAXIMUM_OBJECT_SIZE); + assertTrue(tempByteCount > TEST_PARTSIZE_LONG); + + final TestRunner runner = initRunner(new PutOBSObject()); + runner.setProperty(PutOBSObject.MULTIPART_THRESHOLD, TEST_PARTSIZE_STRING); + runner.setProperty(PutOBSObject.MULTIPART_PART_SIZE, TEST_PARTSIZE_STRING); + + Map attributes = new HashMap<>(); + attributes.put(CoreAttributes.FILENAME.key(), FILE1_NAME); + runner.enqueue(new FileInputStream(tempFile.toFile()), attributes); + + runner.assertValid(); + runner.run(); + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS); + final List successFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + assertEquals(1, successFiles.size()); + final List failureFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_FAILURE); + assertEquals(0, failureFiles.size()); + MockFlowFile ff1 = successFiles.get(0); + assertEquals(OBS_API_METHOD_MULTIPART_UPLOAD, ff1.getAttribute(OBS_API_METHOD_ATTR_KEY)); + assertEquals(FILE1_NAME, ff1.getAttribute(CoreAttributes.FILENAME.key())); + assertEquals(bucketName, ff1.getAttribute(OBS_BUCKET)); + assertEquals(FILE1_NAME, ff1.getAttribute(OBS_OBJECT)); + assertEquals(tempByteCount, ff1.getSize()); + } + + @Test + public void testMultipartLargerThanObjectMaximum() throws IOException { + final String FILE1_NAME = "file1"; + + final byte[] megabyte = new byte[1024 * 1024]; + final Path tempFile = Files.createTempFile("s3mulitpart", "tmp"); + final FileOutputStream tempOut = new FileOutputStream(tempFile.toFile()); + for (int i = 0; i < 305; i++) { + tempOut.write(megabyte); + } + tempOut.close(); + + final TestRunner runner = initRunner(new PutOBSObject()); + runner.setProperty(PutOBSObject.MULTIPART_PART_SIZE, TEST_PARTSIZE_STRING); + runner.setProperty(PutOBSObject.MULTIPART_THRESHOLD, "280 mb"); + + Map attributes = new HashMap<>(); + attributes.put(CoreAttributes.FILENAME.key(), FILE1_NAME); + runner.enqueue(new FileInputStream(tempFile.toFile()), attributes); + + runner.assertValid(); + runner.run(); + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS); + final List successFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + assertEquals(1, successFiles.size()); + final List failureFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_FAILURE); + assertEquals(0, failureFiles.size()); + MockFlowFile ff1 = successFiles.get(0); + assertEquals(FILE1_NAME, ff1.getAttribute(CoreAttributes.FILENAME.key())); + assertEquals(bucketName, ff1.getAttribute(OBS_BUCKET)); + assertEquals(FILE1_NAME, ff1.getAttribute(OBS_OBJECT)); + assertTrue(ff1.getSize() == 305 * 1024 * 1024); + } + + @Test + public void testS3MultipartAgeoff() throws InterruptedException, IOException { + final TestRunner runner = initRunner(new PutOBSObject()); + final ProcessContext context = runner.getProcessContext(); + final PutOBSObject processor = (PutOBSObject)runner.getProcessor(); + // set check interval and age off to minimum values + runner.setProperty(PutOBSObject.MULTIPART_AGEOFF_INTERVAL, "1 milli"); + runner.setProperty(PutOBSObject.MULTIPART_MAX_AGE, "1 milli"); + + // create some dummy uploads + for (Integer i = 0; i < 3; i++) { + final InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest( + bucketName, "file" + i + ".txt"); + assertDoesNotThrow(() -> client.initiateMultipartUpload(initiateRequest)); + } + + // Age off is time dependent, so test has some timing constraints. This + // sleep() delays long enough to satisfy interval and age intervals. + Thread.sleep(2000L); + + // actually initialize the state + MultipartUploadListing uploadList = processor.getAgeOffListAndAgeOffLocalState(context, client, System.currentTimeMillis(), bucketName); + assertEquals(3, uploadList.getMultipartTaskList().size()); + + Thread.sleep(500L); + + // abort a task + MultipartUpload upload0 = uploadList.getMultipartTaskList().get(0); + processor.abortMultipartUpload(client, bucketName, upload0); + uploadList = processor.getAgeOffListAndAgeOffLocalState(context, client, System.currentTimeMillis(), bucketName); + // ageOff list after abort a task + assertEquals(2, uploadList.getMultipartTaskList().size()); + + Thread.sleep(500L); + + final Map attrs = new HashMap<>(); + attrs.put("filename", "test-upload.txt"); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + runner.run(); + + uploadList = processor.getAgeOffListAndAgeOffLocalState(context, client, System.currentTimeMillis(), bucketName); + assertEquals(0, uploadList.getMultipartTaskList().size()); + } + + @Test + public void testEncryptionServiceWithServerSideKMSEncryptionStrategyUsingSingleUpload() throws IOException, InitializationException { + byte[] smallData = Files.readAllBytes(getResourcePath(SAMPLE_FILE_RESOURCE_NAME)); + testEncryptionServiceWithServerSideKMSEncryptionStrategy(smallData); + } + + @Test + public void testEncryptionServiceWithServerSideKMSEncryptionStrategyUsingMultipartUpload() throws IOException, InitializationException { + byte[] largeData = new byte[51 * 1024 * 1024]; + testEncryptionServiceWithServerSideKMSEncryptionStrategy(largeData); + } + + private void testEncryptionServiceWithServerSideKMSEncryptionStrategy(byte[] data) throws IOException, InitializationException { + TestRunner runner = createPutEncryptionTestRunner(StandardOBSEncryptionService.SSE_KMS, kmsKeyId); + + final Map attrs = new HashMap<>(); + attrs.put("filename", "test.txt"); + runner.enqueue(data, attrs); + runner.assertValid(); + runner.run(); + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS); + + List flowFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + assertEquals(1, flowFiles.size()); + assertEquals(0, runner.getFlowFilesForRelationship(PutOBSObject.REL_FAILURE).size()); + MockFlowFile putSuccess = flowFiles.get(0); + assertEquals(putSuccess.getAttribute(OBS_ENCRYPTION_STRATEGY), ObsServiceEncryptionService.STRATEGY_NAME_SSE_KMS); + + MockFlowFile flowFile = fetchEncryptedFlowFile(attrs, StandardOBSEncryptionService.SSE_KMS, kmsKeyId); + flowFile.assertContentEquals(data); + flowFile.assertAttributeEquals(OBS_SSE_ALGORITHM, null); + flowFile.assertAttributeEquals(OBS_ENCRYPTION_STRATEGY, null); + } + + @Test + public void testEncryptionServiceWithServerSideCEncryptionStrategyUsingSingleUpload() throws IOException, InitializationException { + byte[] smallData = Files.readAllBytes(getResourcePath(SAMPLE_FILE_RESOURCE_NAME)); + testEncryptionServiceWithServerSideCEncryptionStrategy(smallData); + } + + @Test + public void testEncryptionServiceWithServerSideCEncryptionStrategyUsingMultipartUpload() throws IOException, InitializationException { + byte[] largeData = new byte[51 * 1024 * 1024]; + testEncryptionServiceWithServerSideCEncryptionStrategy(largeData); + } + + private void testEncryptionServiceWithServerSideCEncryptionStrategy(byte[] data) throws IOException, InitializationException { + TestRunner runner = createPutEncryptionTestRunner(StandardOBSEncryptionService.SSE_C, randomKeyMaterial); + + final Map attrs = new HashMap<>(); + attrs.put("filename", "test.txt"); + runner.enqueue(data, attrs); + runner.assertValid(); + runner.run(); + runner.assertAllFlowFilesTransferred(PutOBSObject.REL_SUCCESS); + + List flowFiles = runner.getFlowFilesForRelationship(PutOBSObject.REL_SUCCESS); + assertEquals(1, flowFiles.size()); + assertEquals(0, runner.getFlowFilesForRelationship(PutOBSObject.REL_FAILURE).size()); + MockFlowFile putSuccess = flowFiles.get(0); + assertEquals(putSuccess.getAttribute(OBS_ENCRYPTION_STRATEGY), ObsServiceEncryptionService.STRATEGY_NAME_SSE_C); + + MockFlowFile flowFile = fetchEncryptedFlowFile(attrs, StandardOBSEncryptionService.SSE_C, randomKeyMaterial); + flowFile.assertContentEquals(data); + // successful fetch does not indicate type of original encryption: + flowFile.assertAttributeEquals(OBS_SSE_ALGORITHM, SSEAlgorithmEnum.AES256.getCode()); + // but it does indicate it via our specific attribute: + flowFile.assertAttributeEquals(OBS_ENCRYPTION_STRATEGY, ObsServiceEncryptionService.STRATEGY_NAME_SSE_C); + } + + private TestRunner createPutEncryptionTestRunner(AllowableValue encryptionStrategy, String keyIdOrMaterial) throws InitializationException { + TestRunner runner = initRunner(new PutOBSObject()); + addEncryptionService(runner, encryptionStrategy, keyIdOrMaterial); + + runner.setProperty(PutOBSObject.MULTIPART_THRESHOLD, "50 MB"); + runner.setProperty(PutOBSObject.MULTIPART_PART_SIZE, "50 MB"); + + return runner; + } + + private MockFlowFile fetchEncryptedFlowFile(Map attributes, AllowableValue encryptionStrategy, String keyIdOrMaterial) throws InitializationException { + TestRunner runner = initRunner(new FetchOBSObject()); + if (StandardOBSEncryptionService.SSE_C.equals(encryptionStrategy)) { + addEncryptionService(runner, encryptionStrategy, keyIdOrMaterial); + } + runner.enqueue(new byte[0], attributes); + runner.run(1); + runner.assertAllFlowFilesTransferred(FetchOBSObject.REL_SUCCESS, 1); + List flowFiles = runner.getFlowFilesForRelationship(FetchOBSObject.REL_SUCCESS); + return flowFiles.get(0); + } + + + private class MockAmazonS3Client extends ObsClient { + MultipartUploadListing listing; + + public MockAmazonS3Client(String endPoint) { + super(endPoint); + } + + public void setListing(MultipartUploadListing newlisting) { + listing = newlisting; + } + + @Override + public MultipartUploadListing listMultipartUploads(ListMultipartUploadsRequest listMultipartUploadsRequest) + throws ObsException { + return listing; + } + } +} diff --git a/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/smn/TestPublishSMNMessage.java b/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/smn/TestPublishSMNMessage.java new file mode 100644 index 0000000000000000000000000000000000000000..42bb843922ec9390960ee2de2cd5ba44c5a4c969 --- /dev/null +++ b/nifi-huawei-processors/src/test/java/org/apache/nifi/processors/huawei/smn/TestPublishSMNMessage.java @@ -0,0 +1,95 @@ +package org.apache.nifi.processors.huawei.smn; + +import com.huaweicloud.sdk.smn.v2.SmnClient; +import com.huaweicloud.sdk.smn.v2.model.PublishMessageRequest; +import com.huaweicloud.sdk.smn.v2.model.PublishMessageResponse; +import org.apache.nifi.processors.huawei.abstractprocessor.AbstractHuaweiProcessor; +import org.apache.nifi.processors.huawei.common.PropertyDescriptors; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestPublishSMNMessage { + + private TestRunner runner = null; + private PublishSMNMessage mockPublishSMNMessage = null; + private SmnClient mockSmnClient = null; + + @BeforeEach + public void setUp() { + + mockSmnClient = Mockito.mock(SmnClient.class); + mockPublishSMNMessage = new PublishSMNMessage() { + @Override + public SmnClient getSmnClient() { return mockSmnClient; } + }; + runner = TestRunners.newTestRunner(mockPublishSMNMessage); + } + + @Test + public void testPublishMessage() { + + runner.setProperty(PropertyDescriptors.ACCESS_KEY, "test-ak"); + runner.setProperty(PropertyDescriptors.SECRET_KEY, "test-sk"); + runner.setProperty(PropertyDescriptors.SMN_REGION, "ap-southeast-3"); + runner.setProperty(PublishSMNMessage.SMN_TOPIC_URN, "urn:smn:ap-southeast-3:test-projectId:AirFlowShowCase"); + runner.setProperty(PublishSMNMessage.SMN_PROJECT_ID, "test-projectId"); + runner.setProperty(PublishSMNMessage.SMN_SUBJECT, "${eval.subject}"); + runner.setProperty(PublishSMNMessage.SMN_MESSAGE, "${eval.message}"); + runner.setValidateExpressionUsage(false); + + final Map flowFileAttributes = new HashMap<>(); + flowFileAttributes.put("eval.subject", "test-subject"); + flowFileAttributes.put("eval.message", "test-message"); + runner.enqueue("test-message", flowFileAttributes); + + PublishMessageResponse mockPublishMessageResponse = new PublishMessageResponse(); + Mockito.when(mockSmnClient.publishMessage(Mockito.any(PublishMessageRequest.class))).thenReturn(mockPublishMessageResponse); + + runner.run(); + + ArgumentCaptor captureRequest = ArgumentCaptor.forClass(PublishMessageRequest.class); + Mockito.verify(mockSmnClient, Mockito.times(1)).publishMessage(captureRequest.capture()); + PublishMessageRequest request = captureRequest.getValue(); + assertEquals("urn:smn:ap-southeast-3:test-projectId:AirFlowShowCase", request.getTopicUrn()); + assertEquals("test-message", request.getBody().getMessage()); + assertEquals("test-subject", request.getBody().getSubject()); + + runner.assertAllFlowFilesTransferred(AbstractHuaweiProcessor.REL_SUCCESS, 1); + } + + @Test + public void testPublishMessageFailure() { + + runner.setProperty(PropertyDescriptors.ACCESS_KEY, "test-ak"); + runner.setProperty(PropertyDescriptors.SECRET_KEY, "test-sk"); + runner.setProperty(PropertyDescriptors.SMN_REGION, "ap-southeast-3"); + runner.setProperty(PublishSMNMessage.SMN_TOPIC_URN, "urn:smn:ap-southeast-3:test-projectId:AirFlowShowCase"); + runner.setProperty(PublishSMNMessage.SMN_PROJECT_ID, "test-projectId"); + runner.setProperty(PublishSMNMessage.SMN_SUBJECT, "${eval.subject}"); + runner.setProperty(PublishSMNMessage.SMN_MESSAGE, "${eval.message}"); + runner.setValidateExpressionUsage(false); + + final Map flowFileAttributes = new HashMap<>(); + flowFileAttributes.put("eval.subject", "test-subject"); + flowFileAttributes.put("eval.message", "test-message"); + runner.enqueue("test-message", flowFileAttributes); + + Mockito.when(mockSmnClient.publishMessage(Mockito.any(PublishMessageRequest.class))).thenThrow(new RuntimeException()); + + runner.run(); + + ArgumentCaptor captureRequest = ArgumentCaptor.forClass(PublishMessageRequest.class); + Mockito.verify(mockSmnClient, Mockito.times(1)).publishMessage(captureRequest.capture()); + + runner.assertAllFlowFilesTransferred(AbstractHuaweiProcessor.REL_FAILURE, 1); + } +} diff --git a/nifi-huawei-processors/src/test/resources/hello.txt b/nifi-huawei-processors/src/test/resources/hello.txt new file mode 100644 index 0000000000000000000000000000000000000000..ee13cb732d05301c7a514512450b572db10293c8 --- /dev/null +++ b/nifi-huawei-processors/src/test/resources/hello.txt @@ -0,0 +1 @@ +Hello, World!! \ No newline at end of file diff --git a/nifi-huawei-service-api-nar/pom.xml b/nifi-huawei-service-api-nar/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..443d2683a4b4bd0e27d23ec6975fd3d5f262201a --- /dev/null +++ b/nifi-huawei-service-api-nar/pom.xml @@ -0,0 +1,45 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-huawei-bundle + 1.18.0 + + + nifi-huawei-service-api-nar + nar + + true + true + + + + + org.apache.nifi + nifi-standard-services-api-nar + 1.18.0 + nar + + + org.apache.nifi + nifi-huawei-service-api + 1.18.0 + + + diff --git a/nifi-huawei-service-api-nar/src/main/resources/META-INF/LICENSE b/nifi-huawei-service-api-nar/src/main/resources/META-INF/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..293a59c46bf56c8a2718e23ff1fe632191bc320e --- /dev/null +++ b/nifi-huawei-service-api-nar/src/main/resources/META-INF/LICENSE @@ -0,0 +1,232 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +APACHE NIFI SUBCOMPONENTS: + +The Apache NiFi project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + + The binary distribution of this product bundles 'Bouncy Castle JDK 1.5' + under an MIT style license. + + Copyright (c) 2000 - 2015 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. diff --git a/nifi-huawei-service-api/pom.xml b/nifi-huawei-service-api/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..b281280a21887708433a03b9c20c0d1f8d1c88bd --- /dev/null +++ b/nifi-huawei-service-api/pom.xml @@ -0,0 +1,35 @@ + + + + + nifi-huawei-bundle + org.apache.nifi + 1.18.0 + + 4.0.0 + + nifi-huawei-service-api + + + org.apache.nifi + nifi-api + + + com.huaweicloud + esdk-obs-java-bundle + + + diff --git a/nifi-huawei-service-api/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/service/HuaweiCredentialsProviderService.java b/nifi-huawei-service-api/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/service/HuaweiCredentialsProviderService.java new file mode 100644 index 0000000000000000000000000000000000000000..22496a8087f9708821c0bd75a22f12bf61818bd1 --- /dev/null +++ b/nifi-huawei-service-api/src/main/java/org/apache/nifi/processors/huawei/credentials/provider/service/HuaweiCredentialsProviderService.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.credentials.provider.service; + +import com.obs.services.IObsCredentialsProvider; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.processor.exception.ProcessException; + +/** + * HuaweiCredentialsProviderService interface to support getting HuaweiCredentialsProvider used for instantiating + * huawei yun clients + + */ +@Tags({"huawei", "security", "credentials", "provider", "session"}) +@CapabilityDescription("Provides HuaweiCredentialsProvider.") +public interface HuaweiCredentialsProviderService extends ControllerService { + + /** + * Get credentials provider + * @return credentials provider + * @throws ProcessException process exception in case there is problem in getting credentials provider + */ + IObsCredentialsProvider getCredentialsProvider() throws ProcessException; +} diff --git a/nifi-huawei-service-api/src/main/java/org/apache/nifi/processors/huawei/obs/ObsServiceEncryptionService.java b/nifi-huawei-service-api/src/main/java/org/apache/nifi/processors/huawei/obs/ObsServiceEncryptionService.java new file mode 100644 index 0000000000000000000000000000000000000000..a6e7166eea203201a5fab9b5e63a0b023fc132c6 --- /dev/null +++ b/nifi-huawei-service-api/src/main/java/org/apache/nifi/processors/huawei/obs/ObsServiceEncryptionService.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.huawei.obs; + +import com.obs.services.model.*; +import org.apache.nifi.controller.ControllerService; + +/** + * This interface defines how clients interact with an obs encryption service. + */ +public interface ObsServiceEncryptionService extends ControllerService { + + String STRATEGY_NAME_NONE = "NONE"; + String STRATEGY_NAME_SSE_KMS = "SSE_KMS"; + String STRATEGY_NAME_SSE_C = "SSE_C"; + + /** + * Configure a {@link PutObjectRequest} for encryption. + * @param request the request to configure. + * @param objectMetadata the request metadata to configure. + */ + void configurePutObjectRequest(PutObjectRequest request, ObjectMetadata objectMetadata); + + /** + * Configure an {@link InitiateMultipartUploadRequest} for encryption. + * @param request the request to configure. + * @param objectMetadata the request metadata to configure. + */ + void configureInitiateMultipartUploadRequest(InitiateMultipartUploadRequest request, ObjectMetadata objectMetadata); + + /** + * Configure a {@link GetObjectRequest} for encryption. + * @param request the request to configure. + * @param objectMetadata the request metadata to configure. + */ + void configureGetObjectRequest(GetObjectRequest request, ObjectMetadata objectMetadata); + + /** + * Configure an {@link UploadPartRequest} for encryption. + * @param request the request to configure. + * @param objectMetadata the request metadata to configure. + */ + void configureUploadPartRequest(UploadPartRequest request, ObjectMetadata objectMetadata); + + /** + * @return The name of the encryption strategy associated with the service. + */ + String getStrategyName(); +} diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..2d86e61052c860a8795e2dcc8c6dec300bf4a7a1 --- /dev/null +++ b/pom.xml @@ -0,0 +1,55 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-nar-bundles + 1.18.0 + + + nifi-huawei-bundle + pom + + + nifi-huawei-processors + nifi-huawei-nar + nifi-huawei-service-api + nifi-huawei-service-api-nar + + + + + com.huaweicloud + esdk-obs-java-bundle + 3.22.3 + compile + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + + + + +