pax_global_header00006660000000000000000000000064151024417000014504gustar00rootroot0000000000000052 comment=83bf4e2234fb4166e94000b5e718919cec4c2059 minio-go-7.0.97/000077500000000000000000000000001510244170000133265ustar00rootroot00000000000000minio-go-7.0.97/.github/000077500000000000000000000000001510244170000146665ustar00rootroot00000000000000minio-go-7.0.97/.github/workflows/000077500000000000000000000000001510244170000167235ustar00rootroot00000000000000minio-go-7.0.97/.github/workflows/go-windows.yml000066400000000000000000000031571510244170000215510ustar00rootroot00000000000000name: Build (Windows) on: pull_request: branches: - master # This ensures that previous jobs for the PR are canceled when the PR is # updated. concurrency: group: ${{ github.workflow }}-${{ github.head_ref }} cancel-in-progress: true jobs: build: name: Test on Go ${{ matrix.go-version }} and ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: go-version: [1.23.x, 1.24.x] os: [windows-latest] steps: - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }} uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} id: go - name: Check out code into the Go module directory uses: actions/checkout@v4 - name: Build on ${{ matrix.os }} env: MINT_MODE: core SERVER_ENDPOINT: localhost:9000 ACCESS_KEY: minioadmin SECRET_KEY: minioadmin ENABLE_HTTPS: 1 MINIO_KMS_MASTER_KEY: my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574 MINIO_CI_CD: true MINT_NO_FULL_OBJECT: true run: | New-Item -ItemType Directory -Path "$env:temp/certs-dir" Copy-Item -Path testcerts\* -Destination "$env:temp/certs-dir" Invoke-WebRequest -Uri https://dl.minio.io/server/minio/release/windows-amd64/minio.exe -OutFile $HOME/minio.exe Start-Process -NoNewWindow -FilePath "$HOME/minio.exe" -ArgumentList "-S", "$env:temp/certs-dir", "server", "$env:temp/fs{1...4}" $env:SSL_CERT_FILE = "$env:temp/certs-dir/public.crt" go run functional_tests.go minio-go-7.0.97/.github/workflows/go.yml000066400000000000000000000027571510244170000200660ustar00rootroot00000000000000name: Build (Linux) on: pull_request: branches: - master # This ensures that previous jobs for the PR are canceled when the PR is # updated. concurrency: group: ${{ github.workflow }}-${{ github.head_ref }} cancel-in-progress: true jobs: build: name: Test on Go ${{ matrix.go-version }} and ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: go-version: [1.23.x, 1.24.x] os: [ubuntu-latest] steps: - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }} uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} id: go - name: Check out code into the Go module directory uses: actions/checkout@v4 - name: Build on ${{ matrix.os }} env: MINT_MODE: full SERVER_ENDPOINT: localhost:9000 ACCESS_KEY: minioadmin SECRET_KEY: minioadmin ENABLE_HTTPS: 1 MINIO_KMS_SECRET_KEY: my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= SSL_CERT_FILE: /tmp/certs-dir/public.crt MINIO_CI_CD: true MINT_NO_FULL_OBJECT: true run: | sudo apt update -y sudo apt install devscripts -y wget -O /tmp/minio https://dl.minio.io/server/minio/release/linux-amd64/minio chmod +x /tmp/minio mkdir -p /tmp/certs-dir cp testcerts/* /tmp/certs-dir /tmp/minio server --quiet -S /tmp/certs-dir /tmp/fs{1...4} & make minio-go-7.0.97/.github/workflows/vulncheck.yml000066400000000000000000000012031510244170000214240ustar00rootroot00000000000000name: VulnCheck on: pull_request: branches: - master - main push: branches: - master - main jobs: vulncheck: name: Analysis runs-on: ubuntu-latest strategy: matrix: go-version: [ 1.24.x ] steps: - name: Check out code into the Go module directory uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} check-latest: true - name: Get govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest shell: bash - name: Run govulncheck run: govulncheck ./... shell: bash minio-go-7.0.97/.gitignore000066400000000000000000000000701510244170000153130ustar00rootroot00000000000000*~ *.test validator golangci-lint functional_tests .ideaminio-go-7.0.97/.golangci.yml000066400000000000000000000027531510244170000157210ustar00rootroot00000000000000version: "2" linters: disable-all: true enable: - durationcheck - gocritic - gomodguard - govet - ineffassign - misspell - revive - staticcheck - unconvert - unused - usetesting - whitespace settings: misspell: locale: US staticcheck: checks: - all - -SA1008 - -SA1019 - -SA4000 - -SA9004 - -ST1000 - -ST1005 - -ST1016 - -ST1021 - -ST1020 - -U1000 exclusions: generated: lax rules: - path: (.+)\.go$ text: "empty-block:" - path: (.+)\.go$ text: "unused-parameter:" - path: (.+)\.go$ text: "dot-imports:" - path: (.+)\.go$ text: "singleCaseSwitch: should rewrite switch statement to if statement" - path: (.+)\.go$ text: "unlambda: replace" - path: (.+)\.go$ text: "captLocal:" - path: (.+)\.go$ text: "should have a package comment" - path: (.+)\.go$ text: "ifElseChain:" - path: (.+)\.go$ text: "elseif:" - path: (.+)\.go$ text: "Error return value of" - path: (.+)\.go$ text: "unnecessary conversion" - path: (.+)\.go$ text: "Error return value is not checked" issues: max-issues-per-linter: 100 max-same-issues: 100 formatters: enable: - gofumpt - goimports exclusions: generated: lax paths: - third_party$ - builtin$ - examples$ minio-go-7.0.97/200OKwithError_test.go000066400000000000000000000143441510244170000173630ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "fmt" "net/http" "net/http/httptest" "net/url" "testing" "time" "github.com/minio/minio-go/v7/pkg/credentials" ) func Test200MultipartUploadWithSpaces(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(``)) w.(http.Flusher).Flush() for i := 0; i < 10; i++ { time.Sleep(time.Second) w.Write([]byte(" ")) w.(http.Flusher).Flush() } w.Write([]byte(`http://random/bucket/objectbucketobject"2b3ffa539769372e2df9553358fe26b2-2"`)) })) srv, err := url.Parse(ts.URL) if err != nil { t.Fatal(err) } // Instantiate new minio client object. core, err := NewCore( srv.Host, &Options{ Creds: credentials.NewStaticV4("foo", "foo12345", ""), Secure: srv.Scheme == "https", }) if err != nil { t.Fatal("Error:", err) } parts := []CompletePart{ {PartNumber: 1, ETag: "b386a859d8a22ff986c0b1252be34658"}, {PartNumber: 2, ETag: "78c577a580bbbba92845789cda1fa932"}, } foundUploadInfo, err := core.CompleteMultipartUpload(context.Background(), "bucket", "object", "jY1M2U5NWMtZGY2OC00ZjYyLTljZGYtYmZlOWEzODM3MDMwLjlmZWY5OGNlLWQ1Y2EtNDgwMC04N2Y4LWZkNTNkMDM4ZDdiMXgxNzQ4NjA0NzI0NzE4NjU3MTY3", parts, PutObjectOptions{}, ) if err != nil { t.Fatal("Error:", err) } expectedUploadInfo := UploadInfo{ Bucket: "bucket", Key: "object", ETag: "2b3ffa539769372e2df9553358fe26b2-2", Location: "http://random/bucket/object", } if foundUploadInfo != expectedUploadInfo { t.Fatalf("Unexpected upload info, expected: `%v`, found: `%v`", expectedUploadInfo, foundUploadInfo) } } func Test200MultipartUploadWithError(t *testing.T) { const maxRetries = 3 retries := maxRetries ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { retries-- w.Write([]byte(``)) w.(http.Flusher).Flush() for i := 0; i < 5; i++ { time.Sleep(time.Second) w.Write([]byte(" ")) w.(http.Flusher).Flush() } w.Write([]byte(`SlowDownWriteResource requested is unwritable, please reduce your request rateobjectbucket/bucket/object18413E84F6C3061349371f38c0d7ec74eae2befc695360a3dfece04732914e58a4281759cd2eba4f`)) })) srv, err := url.Parse(ts.URL) if err != nil { t.Fatal(err) } // Instantiate new minio client object. core, err := NewCore( srv.Host, &Options{ Creds: credentials.NewStaticV4("foo", "foo12345", ""), Secure: srv.Scheme == "https", Region: "us-east-1", MaxRetries: retries, }) if err != nil { t.Fatal("Error:", err) } parts := []CompletePart{ {PartNumber: 1, ETag: "b386a859d8a22ff986c0b1252be34658"}, {PartNumber: 2, ETag: "78c577a580bbbba92845789cda1fa932"}, } _, err = core.CompleteMultipartUpload(context.Background(), "bucket", "object", "jY1M2U5NWMtZGY2OC00ZjYyLTljZGYtYmZlOWEzODM3MDMwLjlmZWY5OGNlLWQ1Y2EtNDgwMC04N2Y4LWZkNTNkMDM4ZDdiMXgxNzQ4NjA0NzI0NzE4NjU3MTY3", parts, PutObjectOptions{}, ) if err == nil { t.Fatal("CompleteMultipartUpload() returned , which is unexpected") } expectedErrorMsg := "Resource requested is unwritable, please reduce your request rate" if err.Error() != expectedErrorMsg { t.Fatalf("Unexpected returned error, expected: `%v`, found: `%v`", expectedErrorMsg, err.Error()) } if retries != 0 { t.Fatalf("CompleteMultipart request was not retried enough times, expected: %d, found: %d", maxRetries, retries) } } func Test200DeleteObjectsWithError(t *testing.T) { const maxRetries = 3 retries := maxRetries ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { retries-- w.Write([]byte(``)) w.(http.Flusher).Flush() for i := 0; i < 5; i++ { time.Sleep(time.Second) w.Write([]byte(" ")) w.(http.Flusher).Flush() } w.Write([]byte(`SlowDownWriteResource requested is unwritable, please reduce your request rateobjectbucket/bucket/object18413E84F6C3061349371f38c0d7ec74eae2befc695360a3dfece04732914e58a4281759cd2eba4f`)) })) srv, err := url.Parse(ts.URL) if err != nil { t.Fatal(err) } // Instantiate new minio client object. core, err := NewCore( srv.Host, &Options{ Creds: credentials.NewStaticV4("foo", "foo12345", ""), Secure: srv.Scheme == "https", Region: "us-east-1", MaxRetries: retries, }) if err != nil { t.Fatal("Error:", err) } // core.TraceOn(os.Stderr) objs := make(chan ObjectInfo, 1000) for i := range 1000 { objs <- ObjectInfo{Key: fmt.Sprintf("obj-%d", i)} } close(objs) delErrCh := core.RemoveObjects(context.Background(), "bucket", objs, RemoveObjectsOptions{}) delErr := <-delErrCh err = delErr.Err if err == nil { t.Fatal("RemoveObjects() returned , which is unexpected") } expectedErrorMsg := "Resource requested is unwritable, please reduce your request rate" if err.Error() != expectedErrorMsg { t.Fatalf("Unexpected returned error, expected: `%v`, found: `%v`", expectedErrorMsg, err.Error()) } if retries != 0 { t.Fatalf("RemoveObjects() request was not retried enough times, expected: %d, found: %d", maxRetries, retries) } } minio-go-7.0.97/AGENTS.md000077700000000000000000000000001510244170000161032CLAUDE.mdustar00rootroot00000000000000minio-go-7.0.97/CLAUDE.md000066400000000000000000000076051510244170000146150ustar00rootroot00000000000000CLAUDE.md ========= This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. Commands -------- ### Testing ```bash # Run all tests with race detection (requires MinIO server at localhost:9000) SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minioadmin SECRET_KEY=minioadmin ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... # Run tests without race detection go test ./... # Run short tests only (no functional tests) go test -short -race ./... # Run functional tests go build -race functional_tests.go SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minioadmin SECRET_KEY=minioadmin ENABLE_HTTPS=1 MINT_MODE=full ./functional_tests # Run functional tests without TLS SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minioadmin SECRET_KEY=minioadmin ENABLE_HTTPS=0 MINT_MODE=full ./functional_tests ``` ### Linting and Code Quality ```bash # Run all checks (lint, vet, test, examples, functional tests) make checks # Run linter only make lint # Run vet and staticcheck make vet # Alternative: run golangci-lint directly golangci-lint run --timeout=5m --config ./.golangci.yml ``` ### Building Examples ```bash # Build all examples make examples # Build a specific example cd examples/s3 && go build -mod=mod putobject.go ``` Architecture ------------ ### Core Client Structure The MinIO Go SDK is organized around a central `Client` struct (api.go:52) that implements Amazon S3 compatible methods. Key architectural patterns: 1. **Modular API Organization**: API methods are split into logical files: - `api-bucket-*.go`: Bucket operations (lifecycle, encryption, versioning, etc.) - `api-object-*.go`: Object operations (legal hold, retention, tagging, etc.) - `api-get-*.go`, `api-put-*.go`: GET and PUT operations - `api-list.go`: Listing operations - `api-stat.go`: Status/info operations 2. **Credential Management**: The `pkg/credentials/` package provides various credential providers: - Static credentials - Environment variables (AWS/MinIO) - IAM roles - STS (Security Token Service) variants - File-based credentials - Chain provider for fallback mechanisms 3. **Request Signing**: The `pkg/signer/` package handles AWS signature versions: - V2 signatures (legacy) - V4 signatures (standard) - Streaming signatures for large uploads 4. **Transport Layer**: Custom HTTP transport with: - Retry logic with configurable max retries - Health status monitoring - Tracing support via httptrace - Bucket location caching (`bucketLocCache`\) - Session caching for credentials 5. **Helper Packages**: - `pkg/encrypt/`: Server-side encryption utilities - `pkg/notification/`: Event notification handling - `pkg/policy/`: Bucket policy management - `pkg/lifecycle/`: Object lifecycle rules - `pkg/tags/`: Object and bucket tagging - `pkg/s3utils/`: S3 utility functions - `pkg/kvcache/`: Key-value caching - `pkg/singleflight/`: Deduplication of concurrent requests ### Testing Strategy - Unit tests alongside implementation files (`*_test.go`\) - Comprehensive functional tests in `functional_tests.go` requiring a live MinIO server - Example programs in `examples/` directory demonstrating API usage - Build tag `//go:build mint` for integration tests ### Error Handling - Custom error types in `api-error-response.go` - HTTP status code mapping - Retry logic for transient failures - Detailed error context preservation Important Patterns ------------------ 1. **Context Usage**: All API methods accept `context.Context` for cancellation and timeout control 2. **Options Pattern**: Methods use Options structs for optional parameters (e.g., `PutObjectOptions`, `GetObjectOptions`\) 3. **Streaming Support**: Large file operations use io.Reader/Writer interfaces for memory efficiency 4. **Bucket Lookup Types**: Supports both path-style and virtual-host-style S3 URLs 5. **MD5/SHA256 Hashing**: Configurable hash functions for integrity checks via `md5Hasher` and `sha256Hasher` minio-go-7.0.97/CNAME000066400000000000000000000000171510244170000140720ustar00rootroot00000000000000minio-go.min.iominio-go-7.0.97/CONTRIBUTING.md000066400000000000000000000024101510244170000155540ustar00rootroot00000000000000### Developer Guidelines `minio-go` welcomes your contribution. To make the process as seamless as possible, we ask for the following: - Go ahead and fork the project and make your changes. We encourage pull requests to discuss code changes. - Fork it - Create your feature branch (git checkout -b my-new-feature) - Commit your changes (git commit -am 'Add some feature') - Push to the branch (git push origin my-new-feature) - Create new Pull Request - When you're ready to create a pull request, be sure to: - Have test cases for the new code. If you have questions about how to do it, please ask in your pull request. - Run `go fmt` - Squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request. - Make sure `go test -race ./...` and `go build` completes. NOTE: go test runs functional tests and requires you to have a AWS S3 account. Set them as environment variables`ACCESS_KEY` and `SECRET_KEY`. To run shorter version of the tests please use `go test -short -race ./...` - Read [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project - `minio-go` project is strictly conformant with Golang style - if you happen to observe offending code, please feel free to send a pull request minio-go-7.0.97/CREDITS000066400000000000000000001610201510244170000143460ustar00rootroot00000000000000Go (the standard library) https://golang.org/ ---------------------------------------------------------------- Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================ github.com/davecgh/go-spew https://github.com/davecgh/go-spew ---------------------------------------------------------------- ISC License Copyright (c) 2012-2016 Dave Collins Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ================================================================ github.com/dustin/go-humanize https://github.com/dustin/go-humanize ---------------------------------------------------------------- Copyright (c) 2005-2008 Dustin Sallings Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================================ github.com/goccy/go-json https://github.com/goccy/go-json ---------------------------------------------------------------- MIT License Copyright (c) 2020 Masaaki Goshima Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================================ github.com/google/uuid https://github.com/google/uuid ---------------------------------------------------------------- Copyright (c) 2009,2014 Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================ github.com/klauspost/compress https://github.com/klauspost/compress ---------------------------------------------------------------- Copyright (c) 2012 The Go Authors. All rights reserved. Copyright (c) 2019 Klaus Post. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------------------ Files: gzhttp/* Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2016-2017 The New York Times Company Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------ Files: s2/cmd/internal/readahead/* The MIT License (MIT) Copyright (c) 2015 Klaus Post Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------- Files: snappy/* Files: internal/snapref/* Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ----------------- Files: s2/cmd/internal/filepathx/* Copyright 2016 The filepathx Authors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================================ github.com/klauspost/cpuid/v2 https://github.com/klauspost/cpuid/v2 ---------------------------------------------------------------- The MIT License (MIT) Copyright (c) 2015 Klaus Post Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================================ github.com/minio/md5-simd https://github.com/minio/md5-simd ---------------------------------------------------------------- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================================ github.com/pmezard/go-difflib https://github.com/pmezard/go-difflib ---------------------------------------------------------------- Copyright (c) 2013, Patrick Mezard All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. The names of its contributors may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================ github.com/rs/xid https://github.com/rs/xid ---------------------------------------------------------------- Copyright (c) 2015 Olivier Poitrey Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================================ github.com/stretchr/testify https://github.com/stretchr/testify ---------------------------------------------------------------- MIT License Copyright (c) 2012-2018 Mat Ryer and Tyler Bunnell Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================================ golang.org/x/crypto https://golang.org/x/crypto ---------------------------------------------------------------- Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================ golang.org/x/net https://golang.org/x/net ---------------------------------------------------------------- Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================ golang.org/x/sys https://golang.org/x/sys ---------------------------------------------------------------- Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================ golang.org/x/text https://golang.org/x/text ---------------------------------------------------------------- Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================ gopkg.in/ini.v1 https://gopkg.in/ini.v1 ---------------------------------------------------------------- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: You must give any other recipients of the Work or Derivative Works a copy of this License; and You must cause any modified files to carry prominent notices stating that You changed the files; and You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2014 Unknwon Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================================ minio-go-7.0.97/LICENSE000066400000000000000000000261361510244170000143430ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. minio-go-7.0.97/MAINTAINERS.md000066400000000000000000000023221510244170000154210ustar00rootroot00000000000000For maintainers only ==================== Responsibilities ---------------- Please go through this link [Maintainer Responsibility](https://gist.github.com/abperiasamy/f4d9b31d3186bbd26522) ### Making new releases Tag and sign your release commit, additionally this step requires you to have access to MinIO's trusted private key. ```sh $ export GNUPGHOME=/media/${USER}/minio/trusted $ git tag -s 4.0.0 $ git push $ git push --tags ``` ### Update version Once release has been made update `libraryVersion` constant in `api.go` to next to be released version. ```sh $ grep libraryVersion api.go libraryVersion = "4.0.1" ``` Commit your changes ``` $ git commit -a -m "Update version for next release" --author "MinIO Trusted " ``` ### Announce Announce new release by adding release notes at https://github.com/minio/minio-go/releases from `trusted@min.io` account. Release notes requires two sections `highlights` and `changelog`. Highlights is a bulleted list of salient features in this release and Changelog contains list of all commits since the last release. To generate `changelog` ```sh $ git log --no-color --pretty=format:'-%d %s (%cr) <%an>' .. ``` minio-go-7.0.97/Makefile000066400000000000000000000033561510244170000147750ustar00rootroot00000000000000GOPATH := $(shell go env GOPATH) TMPDIR := $(shell mktemp -d) all: checks .PHONY: examples docs checks: lint vet test examples functional-test lint: @mkdir -p ${GOPATH}/bin @echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin @echo "Running $@ check" @GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean @GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml vet: @GO111MODULE=on go vet ./... @echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest ${GOPATH}/bin/staticcheck -tests=false -checks="all,-ST1000,-ST1003,-ST1016,-ST1020,-ST1021,-ST1022,-ST1023,-ST1005" test: @GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minioadmin SECRET_KEY=minioadmin ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./... examples: @echo "Building s3 examples" @cd ./examples/s3 && $(foreach v,$(wildcard examples/s3/*.go),go build -mod=mod -o ${TMPDIR}/$(basename $(v)) $(notdir $(v)) || exit 1;) @echo "Building minio examples" @cd ./examples/minio && $(foreach v,$(wildcard examples/minio/*.go),go build -mod=mod -o ${TMPDIR}/$(basename $(v)) $(notdir $(v)) || exit 1;) functional-test: @GO111MODULE=on go build -race functional_tests.go @SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minioadmin SECRET_KEY=minioadmin ENABLE_HTTPS=1 MINT_MODE=full ./functional_tests functional-test-notls: @GO111MODULE=on go build -race functional_tests.go @SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minioadmin SECRET_KEY=minioadmin ENABLE_HTTPS=0 MINT_MODE=full ./functional_tests clean: @echo "Cleaning up all the generated files" @find . -name '*.test' | xargs rm -fv @find . -name '*~' | xargs rm -fv minio-go-7.0.97/NOTICE000066400000000000000000000005511510244170000142330ustar00rootroot00000000000000MinIO Cloud Storage, (C) 2014-2020 MinIO, Inc. This product includes software developed at MinIO, Inc. (https://min.io/). The MinIO project contains unmodified/modified subcomponents too with separate copyright notices and license terms. Your use of the source code for these subcomponents is subject to the terms and conditions of Apache License Version 2.0 minio-go-7.0.97/README.md000066400000000000000000000343141510244170000146120ustar00rootroot00000000000000MinIO Go Client SDK for Amazon S3 Compatible Cloud Storage [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Sourcegraph](https://sourcegraph.com/github.com/minio/minio-go/-/badge.svg)](https://sourcegraph.com/github.com/minio/minio-go?badge) [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/minio/minio-go/blob/master/LICENSE) ================================================================================================================================================================================================================================================================================================================================================================================================================== The MinIO Go Client SDK provides straightforward APIs to access any Amazon S3 compatible object storage. This Quickstart Guide covers how to install the MinIO client SDK, connect to MinIO, and create a sample file uploader. For a complete list of APIs and examples, see the [godoc documentation](https://pkg.go.dev/github.com/minio/minio-go/v7) or [Go Client API Reference](https://min.io/docs/minio/linux/developers/go/API.html). These examples presume a working [Go development environment](https://golang.org/doc/install) and the [MinIO `mc` command line tool](https://min.io/docs/minio/linux/reference/minio-mc.html). Download from Github -------------------- From your project directory: ```sh go get github.com/minio/minio-go/v7 ``` Initialize a MinIO Client Object -------------------------------- The MinIO client requires the following parameters to connect to an Amazon S3 compatible object storage: | Parameter | Description | |-------------------|------------------------------------------------------------| | `endpoint` | URL to object storage service. | | `_minio.Options_` | All the options such as credentials, custom transport etc. | ```go package main import ( "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { endpoint := "play.min.io" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true // Initialize minio client object. minioClient, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), Secure: useSSL, }) if err != nil { log.Fatalln(err) } log.Printf("%#v\n", minioClient) // minioClient is now set up } ``` Example - File Uploader ----------------------- This sample code connects to an object storage server, creates a bucket, and uploads a file to the bucket. It uses the MinIO `play` server, a public MinIO cluster located at [https://play.min.io](https://play.min.io). The `play` server runs the latest stable version of MinIO and may be used for testing and development. The access credentials shown in this example are open to the public and all data uploaded to `play` should be considered public and non-protected. ### FileUploader.go This example does the following: - Connects to the MinIO `play` server using the provided credentials. - Creates a bucket named `testbucket`. - Uploads a file named `testdata` from `/tmp`. - Verifies the file was created using `mc ls`. ```go // FileUploader.go MinIO example package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { ctx := context.Background() endpoint := "play.min.io" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true // Initialize minio client object. minioClient, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), Secure: useSSL, }) if err != nil { log.Fatalln(err) } // Make a new bucket called testbucket. bucketName := "testbucket" location := "us-east-1" err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location}) if err != nil { // Check to see if we already own this bucket (which happens if you run this twice) exists, errBucketExists := minioClient.BucketExists(ctx, bucketName) if errBucketExists == nil && exists { log.Printf("We already own %s\n", bucketName) } else { log.Fatalln(err) } } else { log.Printf("Successfully created %s\n", bucketName) } // Upload the test file // Change the value of filePath if the file is in another location objectName := "testdata" filePath := "/tmp/testdata" contentType := "application/octet-stream" // Upload the test file with FPutObject info, err := minioClient.FPutObject(ctx, bucketName, objectName, filePath, minio.PutObjectOptions{ContentType: contentType}) if err != nil { log.Fatalln(err) } log.Printf("Successfully uploaded %s of size %d\n", objectName, info.Size) } ``` **1. Create a test file containing data:** You can do this with `dd` on Linux or macOS systems: ```sh dd if=/dev/urandom of=/tmp/testdata bs=2048 count=10 ``` or `fsutil` on Windows: ```sh fsutil file createnew "C:\Users\\Desktop\sample.txt" 20480 ``` **2. Run FileUploader with the following commands:** ```sh go mod init example/FileUploader go get github.com/minio/minio-go/v7 go get github.com/minio/minio-go/v7/pkg/credentials go run FileUploader.go ``` The output resembles the following: ```sh 2023/11/01 14:27:55 Successfully created testbucket 2023/11/01 14:27:55 Successfully uploaded testdata of size 20480 ``` **3. Verify the Uploaded File With `mc ls`:** ```sh mc ls play/testbucket [2023-11-01 14:27:55 UTC] 20KiB STANDARD TestDataFile ``` API Reference ------------- The full API Reference is available here. - [Complete API Reference](https://min.io/docs/minio/linux/developers/go/API.html) ### API Reference : Bucket Operations - [`MakeBucket`](https://min.io/docs/minio/linux/developers/go/API.html#MakeBucket) - [`ListBuckets`](https://min.io/docs/minio/linux/developers/go/API.html#ListBuckets) - [`BucketExists`](https://min.io/docs/minio/linux/developers/go/API.html#BucketExists) - [`RemoveBucket`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveBucket) - [`ListObjects`](https://min.io/docs/minio/linux/developers/go/API.html#ListObjects) - [`ListIncompleteUploads`](https://min.io/docs/minio/linux/developers/go/API.html#ListIncompleteUploads) ### API Reference : Bucket policy Operations - [`SetBucketPolicy`](https://min.io/docs/minio/linux/developers/go/API.html#SetBucketPolicy) - [`GetBucketPolicy`](https://min.io/docs/minio/linux/developers/go/API.html#GetBucketPolicy) ### API Reference : Bucket notification Operations - [`SetBucketNotification`](https://min.io/docs/minio/linux/developers/go/API.html#SetBucketNotification) - [`GetBucketNotification`](https://min.io/docs/minio/linux/developers/go/API.html#GetBucketNotification) - [`RemoveAllBucketNotification`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveAllBucketNotification) - [`ListenBucketNotification`](https://min.io/docs/minio/linux/developers/go/API.html#ListenBucketNotification) (MinIO Extension) - [`ListenNotification`](https://min.io/docs/minio/linux/developers/go/API.html#ListenNotification) (MinIO Extension) ### API Reference : File Object Operations - [`FPutObject`](https://min.io/docs/minio/linux/developers/go/API.html#FPutObject) - [`FGetObject`](https://min.io/docs/minio/linux/developers/go/API.html#FGetObject) ### API Reference : Object Operations - [`GetObject`](https://min.io/docs/minio/linux/developers/go/API.html#GetObject) - [`PutObject`](https://min.io/docs/minio/linux/developers/go/API.html#PutObject) - [`PutObjectStreaming`](https://min.io/docs/minio/linux/developers/go/API.html#PutObjectStreaming) - [`StatObject`](https://min.io/docs/minio/linux/developers/go/API.html#StatObject) - [`CopyObject`](https://min.io/docs/minio/linux/developers/go/API.html#CopyObject) - [`RemoveObject`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveObject) - [`RemoveObjects`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveObjects) - [`RemoveIncompleteUpload`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveIncompleteUpload) - [`SelectObjectContent`](https://min.io/docs/minio/linux/developers/go/API.html#SelectObjectContent) ### API Reference : Presigned Operations - [`PresignedGetObject`](https://min.io/docs/minio/linux/developers/go/API.html#PresignedGetObject) - [`PresignedPutObject`](https://min.io/docs/minio/linux/developers/go/API.html#PresignedPutObject) - [`PresignedHeadObject`](https://min.io/docs/minio/linux/developers/go/API.html#PresignedHeadObject) - [`PresignedPostPolicy`](https://min.io/docs/minio/linux/developers/go/API.html#PresignedPostPolicy) ### API Reference : Client custom settings - [`SetAppInfo`](https://min.io/docs/minio/linux/developers/go/API.html#SetAppInfo) - [`TraceOn`](https://min.io/docs/minio/linux/developers/go/API.html#TraceOn) - [`TraceOff`](https://min.io/docs/minio/linux/developers/go/API.html#TraceOff) Full Examples ------------- ### Full Examples : Bucket Operations - [makebucket.go](https://github.com/minio/minio-go/blob/master/examples/s3/makebucket.go) - [listbuckets.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbuckets.go) - [bucketexists.go](https://github.com/minio/minio-go/blob/master/examples/s3/bucketexists.go) - [removebucket.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucket.go) - [listobjects.go](https://github.com/minio/minio-go/blob/master/examples/s3/listobjects.go) - [listobjectsV2.go](https://github.com/minio/minio-go/blob/master/examples/s3/listobjectsV2.go) - [listincompleteuploads.go](https://github.com/minio/minio-go/blob/master/examples/s3/listincompleteuploads.go) ### Full Examples : Bucket policy Operations - [setbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketpolicy.go) - [getbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketpolicy.go) - [listbucketpolicies.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbucketpolicies.go) ### Full Examples : Bucket lifecycle Operations - [setbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketlifecycle.go) - [getbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketlifecycle.go) ### Full Examples : Bucket encryption Operations - [setbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketencryption.go) - [getbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketencryption.go) - [removebucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucketencryption.go) ### Full Examples : Bucket replication Operations - [setbucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketreplication.go) - [getbucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketreplication.go) - [removebucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucketreplication.go) ### Full Examples : Bucket notification Operations - [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go) - [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go) - [removeallbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeallbucketnotification.go) - [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (MinIO Extension) - [listennotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listen-notification.go) (MinIO Extension) ### Full Examples : File Object Operations - [fputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject.go) - [fgetobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fgetobject.go) ### Full Examples : Object Operations - [putobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/putobject.go) - [getobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/getobject.go) - [statobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/statobject.go) - [copyobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/copyobject.go) - [removeobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeobject.go) - [removeincompleteupload.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeincompleteupload.go) - [removeobjects.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeobjects.go) ### Full Examples : Encrypted Object Operations - [put-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/put-encrypted-object.go) - [get-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/get-encrypted-object.go) - [fput-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputencrypted-object.go) ### Full Examples : Presigned Operations - [presignedgetobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedgetobject.go) - [presignedputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedputobject.go) - [presignedheadobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedheadobject.go) - [presignedpostpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedpostpolicy.go) Explore Further --------------- - [Godoc Documentation](https://pkg.go.dev/github.com/minio/minio-go/v7) - [Complete Documentation](https://min.io/docs/minio/kubernetes/upstream/index.html) - [MinIO Go Client SDK API Reference](https://min.io/docs/minio/linux/developers/go/API.html) Contribute ---------- [Contributors Guide](https://github.com/minio/minio-go/blob/master/CONTRIBUTING.md) License ------- This SDK is distributed under the [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0), see [LICENSE](https://github.com/minio/minio-go/blob/master/LICENSE) and [NOTICE](https://github.com/minio/minio-go/blob/master/NOTICE) for more information. minio-go-7.0.97/api-append-object.go000066400000000000000000000164451510244170000171510ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2025 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "errors" "fmt" "io" "net/http" "strconv" "github.com/minio/minio-go/v7/pkg/s3utils" ) // AppendObjectOptions https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-buckets-objects-append.html type AppendObjectOptions struct { // Provide a progress reader to indicate the current append() progress. Progress io.Reader // ChunkSize indicates the maximum append() size, // it is useful when you want to control how much data // per append() you are interested in sending to server // while keeping the input io.Reader of a longer length. ChunkSize uint64 // Aggressively disable sha256 payload, it is automatically // turned-off for TLS supporting endpoints, useful in benchmarks // where you are interested in the peak() numbers. DisableContentSha256 bool customHeaders http.Header checksumType ChecksumType } // Header returns the custom header for AppendObject API func (opts AppendObjectOptions) Header() (header http.Header) { header = make(http.Header) for k, v := range opts.customHeaders { header[k] = v } return header } func (opts *AppendObjectOptions) setWriteOffset(offset int64) { if len(opts.customHeaders) == 0 { opts.customHeaders = make(http.Header) } opts.customHeaders["x-amz-write-offset-bytes"] = []string{strconv.FormatInt(offset, 10)} } func (opts *AppendObjectOptions) setChecksumParams(info ObjectInfo) { if len(opts.customHeaders) == 0 { opts.customHeaders = make(http.Header) } fullObject := info.ChecksumMode == ChecksumFullObjectMode.String() switch { case info.ChecksumCRC32 != "": if fullObject { opts.checksumType = ChecksumFullObjectCRC32 } case info.ChecksumCRC32C != "": if fullObject { opts.checksumType = ChecksumFullObjectCRC32C } case info.ChecksumCRC64NVME != "": // CRC64NVME only has a full object variant // so it does not carry any special full object // modifier opts.checksumType = ChecksumCRC64NVME } } func (opts AppendObjectOptions) validate(c *Client) (err error) { if opts.ChunkSize > maxPartSize { return errInvalidArgument("Append chunkSize cannot be larger than max part size allowed") } switch { case !c.trailingHeaderSupport: return errInvalidArgument("AppendObject() requires Client with TrailingHeaders enabled") case c.overrideSignerType.IsV2(): return errInvalidArgument("AppendObject() cannot be used with v2 signatures") case s3utils.IsGoogleEndpoint(*c.endpointURL): return errInvalidArgument("AppendObject() cannot be used with GCS endpoints") } return nil } // appendObjectDo - executes the append object http operation. // NOTE: You must have WRITE permissions on a bucket to add an object to it. func (c *Client) appendObjectDo(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts AppendObjectOptions) (UploadInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return UploadInfo{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return UploadInfo{}, err } // Set headers. customHeader := opts.Header() // Populate request metadata. reqMetadata := requestMetadata{ bucketName: bucketName, objectName: objectName, customHeader: customHeader, contentBody: reader, contentLength: size, streamSha256: !opts.DisableContentSha256, } if opts.checksumType.IsSet() { reqMetadata.addCrc = &opts.checksumType reqMetadata.customHeader.Set(amzChecksumAlgo, opts.checksumType.String()) if opts.checksumType.FullObjectRequested() { reqMetadata.customHeader.Set(amzChecksumMode, ChecksumFullObjectMode.String()) } } // Execute PUT an objectName. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return UploadInfo{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { return UploadInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) } } h := resp.Header // When AppendObject() is used, S3 Express will return final object size as x-amz-object-size if amzSize := h.Get("x-amz-object-size"); amzSize != "" { size, err = strconv.ParseInt(amzSize, 10, 64) if err != nil { return UploadInfo{}, err } } return UploadInfo{ Bucket: bucketName, Key: objectName, ETag: trimEtag(h.Get("ETag")), Size: size, // Checksum values ChecksumCRC32: h.Get(ChecksumCRC32.Key()), ChecksumCRC32C: h.Get(ChecksumCRC32C.Key()), ChecksumSHA1: h.Get(ChecksumSHA1.Key()), ChecksumSHA256: h.Get(ChecksumSHA256.Key()), ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()), ChecksumMode: h.Get(ChecksumFullObjectMode.Key()), }, nil } // AppendObject - S3 Express Zone https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-buckets-objects-append.html func (c *Client) AppendObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, opts AppendObjectOptions, ) (info UploadInfo, err error) { if objectSize < 0 && opts.ChunkSize == 0 { return UploadInfo{}, errors.New("object size must be provided when no chunk size is provided") } if err = opts.validate(c); err != nil { return UploadInfo{}, err } oinfo, err := c.StatObject(ctx, bucketName, objectName, StatObjectOptions{Checksum: true}) if err != nil { return UploadInfo{}, err } if oinfo.ChecksumMode != "" && oinfo.ChecksumMode != ChecksumFullObjectMode.String() { return UploadInfo{}, fmt.Errorf("Append() is not allowed on objects that are not of FULL_OBJECT checksum type: %s", oinfo.ChecksumMode) } opts.setChecksumParams(oinfo) // set the appropriate checksum params based on the existing object checksum metadata. opts.setWriteOffset(oinfo.Size) // First append must set the current object size as the offset. if opts.ChunkSize > 0 { finalObjSize := int64(-1) if objectSize > 0 { finalObjSize = info.Size + objectSize } totalPartsCount, partSize, lastPartSize, err := OptimalPartInfo(finalObjSize, opts.ChunkSize) if err != nil { return UploadInfo{}, err } buf := make([]byte, partSize) var partNumber int for partNumber = 1; partNumber <= totalPartsCount; partNumber++ { // Proceed to upload the part. if partNumber == totalPartsCount { partSize = lastPartSize } n, err := readFull(reader, buf) if err != nil { return info, err } if n != int(partSize) { return info, io.ErrUnexpectedEOF } rd := newHook(bytes.NewReader(buf[:n]), opts.Progress) uinfo, err := c.appendObjectDo(ctx, bucketName, objectName, rd, partSize, opts) if err != nil { return info, err } opts.setWriteOffset(uinfo.Size) } } rd := newHook(reader, opts.Progress) return c.appendObjectDo(ctx, bucketName, objectName, rd, objectSize, opts) } minio-go-7.0.97/api-bucket-cors.go000066400000000000000000000102711510244170000166460ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2024 MinIO, Inc. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "net/http" "net/url" "github.com/minio/minio-go/v7/pkg/cors" "github.com/minio/minio-go/v7/pkg/s3utils" ) // SetBucketCors sets the Cross-Origin Resource Sharing (CORS) configuration for the bucket. // If corsConfig is nil, the existing CORS configuration will be removed. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - corsConfig: CORS configuration to apply (nil to remove existing configuration) // // Returns an error if the operation fails. func (c *Client) SetBucketCors(ctx context.Context, bucketName string, corsConfig *cors.Config) error { if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } if corsConfig == nil { return c.removeBucketCors(ctx, bucketName) } return c.putBucketCors(ctx, bucketName, corsConfig) } func (c *Client) putBucketCors(ctx context.Context, bucketName string, corsConfig *cors.Config) error { urlValues := make(url.Values) urlValues.Set("cors", "") corsStr, err := corsConfig.ToXML() if err != nil { return err } reqMetadata := requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentBody: bytes.NewReader(corsStr), contentLength: int64(len(corsStr)), contentMD5Base64: sumMD5Base64([]byte(corsStr)), } resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp != nil { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { return httpRespToErrorResponse(resp, bucketName, "") } } return nil } func (c *Client) removeBucketCors(ctx context.Context, bucketName string) error { urlValues := make(url.Values) urlValues.Set("cors", "") resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return err } if resp.StatusCode != http.StatusNoContent { return httpRespToErrorResponse(resp, bucketName, "") } return nil } // GetBucketCors retrieves the Cross-Origin Resource Sharing (CORS) configuration from the bucket. // If no CORS configuration exists, returns nil with no error. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // // Returns the CORS configuration or an error if the operation fails. func (c *Client) GetBucketCors(ctx context.Context, bucketName string) (*cors.Config, error) { if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err } bucketCors, err := c.getBucketCors(ctx, bucketName) if err != nil { errResponse := ToErrorResponse(err) if errResponse.Code == NoSuchCORSConfiguration { return nil, nil } return nil, err } return bucketCors, nil } func (c *Client) getBucketCors(ctx context.Context, bucketName string) (*cors.Config, error) { urlValues := make(url.Values) urlValues.Set("cors", "") resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, // TODO: needed? copied over from other example, but not spec'd in API. }) defer closeResponse(resp) if err != nil { return nil, err } if resp != nil { if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, bucketName, "") } } corsConfig, err := cors.ParseBucketCorsConfig(resp.Body) if err != nil { return nil, err } return corsConfig, nil } minio-go-7.0.97/api-bucket-encryption.go000066400000000000000000000113351510244170000200740ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/xml" "net/http" "net/url" "github.com/minio/minio-go/v7/pkg/s3utils" "github.com/minio/minio-go/v7/pkg/sse" ) // SetBucketEncryption sets the default encryption configuration on an existing bucket. // The encryption configuration specifies the default encryption behavior for objects uploaded to the bucket. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - config: Server-side encryption configuration to apply // // Returns an error if the operation fails or if config is nil. func (c *Client) SetBucketEncryption(ctx context.Context, bucketName string, config *sse.Configuration) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } if config == nil { return errInvalidArgument("configuration cannot be empty") } buf, err := xml.Marshal(config) if err != nil { return err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("encryption", "") // Content-length is mandatory to set a default encryption configuration reqMetadata := requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentBody: bytes.NewReader(buf), contentLength: int64(len(buf)), contentMD5Base64: sumMD5Base64(buf), } // Execute PUT to upload a new bucket default encryption configuration. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucketName, "") } return nil } // RemoveBucketEncryption removes the default encryption configuration from a bucket. // After removal, the bucket will no longer apply default encryption to new objects. // It uses the provided context to control cancellations and timeouts. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // // Returns an error if the operation fails. func (c *Client) RemoveBucketEncryption(ctx context.Context, bucketName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("encryption", "") // DELETE default encryption configuration on a bucket. resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return err } if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { return httpRespToErrorResponse(resp, bucketName, "") } return nil } // GetBucketEncryption retrieves the default encryption configuration from a bucket. // It uses the provided context to control cancellations and timeouts. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // // Returns the bucket's encryption configuration or an error if the operation fails. func (c *Client) GetBucketEncryption(ctx context.Context, bucketName string) (*sse.Configuration, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("encryption", "") // Execute GET on bucket to get the default encryption configuration. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) defer closeResponse(resp) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, bucketName, "") } encryptionConfig := &sse.Configuration{} if err = xmlDecoder(resp.Body, encryptionConfig); err != nil { return nil, err } return encryptionConfig, nil } minio-go-7.0.97/api-bucket-lifecycle.go000066400000000000000000000114131510244170000176360ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/xml" "io" "net/http" "net/url" "time" "github.com/minio/minio-go/v7/pkg/lifecycle" "github.com/minio/minio-go/v7/pkg/s3utils" ) // SetBucketLifecycle set the lifecycle on an existing bucket. func (c *Client) SetBucketLifecycle(ctx context.Context, bucketName string, config *lifecycle.Configuration) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } // If lifecycle is empty then delete it. if config.Empty() { return c.removeBucketLifecycle(ctx, bucketName) } buf, err := xml.Marshal(config) if err != nil { return err } // Save the updated lifecycle. return c.putBucketLifecycle(ctx, bucketName, buf) } // Saves a new bucket lifecycle. func (c *Client) putBucketLifecycle(ctx context.Context, bucketName string, buf []byte) error { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("lifecycle", "") // Content-length is mandatory for put lifecycle request reqMetadata := requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentBody: bytes.NewReader(buf), contentLength: int64(len(buf)), contentMD5Base64: sumMD5Base64(buf), } // Execute PUT to upload a new bucket lifecycle. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp != nil { if resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucketName, "") } } return nil } // Remove lifecycle from a bucket. func (c *Client) removeBucketLifecycle(ctx context.Context, bucketName string) error { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("lifecycle", "") // Execute DELETE on objectName. resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return err } return nil } // GetBucketLifecycle fetch bucket lifecycle configuration func (c *Client) GetBucketLifecycle(ctx context.Context, bucketName string) (*lifecycle.Configuration, error) { lc, _, err := c.GetBucketLifecycleWithInfo(ctx, bucketName) return lc, err } // GetBucketLifecycleWithInfo fetch bucket lifecycle configuration along with when it was last updated func (c *Client) GetBucketLifecycleWithInfo(ctx context.Context, bucketName string) (*lifecycle.Configuration, time.Time, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, time.Time{}, err } bucketLifecycle, updatedAt, err := c.getBucketLifecycle(ctx, bucketName) if err != nil { return nil, time.Time{}, err } config := lifecycle.NewConfiguration() if err = xml.Unmarshal(bucketLifecycle, config); err != nil { return nil, time.Time{}, err } return config, updatedAt, nil } // Request server for current bucket lifecycle. func (c *Client) getBucketLifecycle(ctx context.Context, bucketName string) ([]byte, time.Time, error) { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("lifecycle", "") urlValues.Set("withUpdatedAt", "true") // Execute GET on bucket to get lifecycle. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) defer closeResponse(resp) if err != nil { return nil, time.Time{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { return nil, time.Time{}, httpRespToErrorResponse(resp, bucketName, "") } } lcBytes, err := io.ReadAll(resp.Body) if err != nil { return nil, time.Time{}, err } const minIOLifecycleCfgUpdatedAt = "X-Minio-LifecycleConfig-UpdatedAt" var updatedAt time.Time if timeStr := resp.Header.Get(minIOLifecycleCfgUpdatedAt); timeStr != "" { updatedAt, err = time.Parse(iso8601DateFormat, timeStr) if err != nil { return nil, time.Time{}, err } } return lcBytes, updatedAt, nil } minio-go-7.0.97/api-bucket-notification.go000066400000000000000000000170701510244170000203720ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bufio" "bytes" "context" "encoding/json" "encoding/xml" "net/http" "net/url" "time" "github.com/minio/minio-go/v7/pkg/notification" "github.com/minio/minio-go/v7/pkg/s3utils" ) // SetBucketNotification saves a new bucket notification with a context to control cancellations and timeouts. func (c *Client) SetBucketNotification(ctx context.Context, bucketName string, config notification.Configuration) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("notification", "") notifBytes, err := xml.Marshal(&config) if err != nil { return err } notifBuffer := bytes.NewReader(notifBytes) reqMetadata := requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentBody: notifBuffer, contentLength: int64(len(notifBytes)), contentMD5Base64: sumMD5Base64(notifBytes), contentSHA256Hex: sum256Hex(notifBytes), } // Execute PUT to upload a new bucket notification. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp != nil { if resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucketName, "") } } return nil } // RemoveAllBucketNotification - Remove bucket notification clears all previously specified config func (c *Client) RemoveAllBucketNotification(ctx context.Context, bucketName string) error { return c.SetBucketNotification(ctx, bucketName, notification.Configuration{}) } // GetBucketNotification returns current bucket notification configuration func (c *Client) GetBucketNotification(ctx context.Context, bucketName string) (bucketNotification notification.Configuration, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return notification.Configuration{}, err } return c.getBucketNotification(ctx, bucketName) } // Request server for notification rules. func (c *Client) getBucketNotification(ctx context.Context, bucketName string) (notification.Configuration, error) { urlValues := make(url.Values) urlValues.Set("notification", "") // Execute GET on bucket to list objects. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return notification.Configuration{}, err } return processBucketNotificationResponse(bucketName, resp) } // processes the GetNotification http response from the server. func processBucketNotificationResponse(bucketName string, resp *http.Response) (notification.Configuration, error) { if resp.StatusCode != http.StatusOK { errResponse := httpRespToErrorResponse(resp, bucketName, "") return notification.Configuration{}, errResponse } var bucketNotification notification.Configuration err := xmlDecoder(resp.Body, &bucketNotification) if err != nil { return notification.Configuration{}, err } return bucketNotification, nil } // ListenNotification listen for all events, this is a MinIO specific API func (c *Client) ListenNotification(ctx context.Context, prefix, suffix string, events []string) <-chan notification.Info { return c.ListenBucketNotification(ctx, "", prefix, suffix, events) } // ListenBucketNotification listen for bucket events, this is a MinIO specific API func (c *Client) ListenBucketNotification(ctx context.Context, bucketName, prefix, suffix string, events []string) <-chan notification.Info { notificationInfoCh := make(chan notification.Info, 1) const notificationCapacity = 4 * 1024 * 1024 notificationEventBuffer := make([]byte, notificationCapacity) // Only success, start a routine to start reading line by line. go func(notificationInfoCh chan<- notification.Info) { defer close(notificationInfoCh) // Validate the bucket name. if bucketName != "" { if err := s3utils.CheckValidBucketName(bucketName); err != nil { select { case notificationInfoCh <- notification.Info{ Err: err, }: case <-ctx.Done(): } return } } // Check ARN partition to verify if listening bucket is supported if s3utils.IsAmazonEndpoint(*c.endpointURL) || s3utils.IsGoogleEndpoint(*c.endpointURL) { select { case notificationInfoCh <- notification.Info{ Err: errAPINotSupported("Listening for bucket notification is specific only to `minio` server endpoints"), }: case <-ctx.Done(): } return } // Prepare urlValues to pass into the request on every loop urlValues := make(url.Values) urlValues.Set("ping", "10") urlValues.Set("prefix", prefix) urlValues.Set("suffix", suffix) urlValues["events"] = events // Wait on the jitter retry loop. for range c.newRetryTimerContinous(time.Second, time.Second*30, MaxJitter) { // Execute GET on bucket to list objects. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) if err != nil { select { case notificationInfoCh <- notification.Info{ Err: err, }: case <-ctx.Done(): } return } // Validate http response, upon error return quickly. if resp.StatusCode != http.StatusOK { errResponse := httpRespToErrorResponse(resp, bucketName, "") select { case notificationInfoCh <- notification.Info{ Err: errResponse, }: case <-ctx.Done(): } return } // Initialize a new bufio scanner, to read line by line. bio := bufio.NewScanner(resp.Body) // Use a higher buffer to support unexpected // caching done by proxies bio.Buffer(notificationEventBuffer, notificationCapacity) // Unmarshal each line, returns marshaled values. for bio.Scan() { var notificationInfo notification.Info if err = json.Unmarshal(bio.Bytes(), ¬ificationInfo); err != nil { // Unexpected error during json unmarshal, send // the error to caller for actionable as needed. select { case notificationInfoCh <- notification.Info{ Err: err, }: case <-ctx.Done(): return } closeResponse(resp) continue } // Empty events pinged from the server if len(notificationInfo.Records) == 0 && notificationInfo.Err == nil { continue } // Send notificationInfo select { case notificationInfoCh <- notificationInfo: case <-ctx.Done(): closeResponse(resp) return } } if err = bio.Err(); err != nil { select { case notificationInfoCh <- notification.Info{ Err: err, }: case <-ctx.Done(): return } } // Close current connection before looping further. closeResponse(resp) } }(notificationInfoCh) // Returns the notification info channel, for caller to start reading from. return notificationInfoCh } minio-go-7.0.97/api-bucket-policy.go000066400000000000000000000111311510244170000171730ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "io" "net/http" "net/url" "strings" "github.com/minio/minio-go/v7/pkg/s3utils" ) // SetBucketPolicy sets the access permissions policy on an existing bucket. // The policy should be a valid JSON string that conforms to the IAM policy format. // If policy is an empty string, the existing bucket policy will be removed. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - policy: JSON policy string (empty string to remove existing policy) // // Returns an error if the operation fails. func (c *Client) SetBucketPolicy(ctx context.Context, bucketName, policy string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } // If policy is empty then delete the bucket policy. if policy == "" { return c.removeBucketPolicy(ctx, bucketName) } // Save the updated policies. return c.putBucketPolicy(ctx, bucketName, policy) } // Saves a new bucket policy. func (c *Client) putBucketPolicy(ctx context.Context, bucketName, policy string) error { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("policy", "") reqMetadata := requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentBody: strings.NewReader(policy), contentLength: int64(len(policy)), } // Execute PUT to upload a new bucket policy. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp != nil { if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucketName, "") } } return nil } // Removes all policies on a bucket. func (c *Client) removeBucketPolicy(ctx context.Context, bucketName string) error { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("policy", "") // Execute DELETE on objectName. resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return err } if resp.StatusCode != http.StatusNoContent { return httpRespToErrorResponse(resp, bucketName, "") } return nil } // GetBucketPolicy retrieves the access permissions policy for the bucket. // If no bucket policy exists, returns an empty string with no error. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // // Returns the policy as a JSON string or an error if the operation fails. func (c *Client) GetBucketPolicy(ctx context.Context, bucketName string) (string, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", err } bucketPolicy, err := c.getBucketPolicy(ctx, bucketName) if err != nil { errResponse := ToErrorResponse(err) if errResponse.Code == NoSuchBucketPolicy { return "", nil } return "", err } return bucketPolicy, nil } // Request server for current bucket policy. func (c *Client) getBucketPolicy(ctx context.Context, bucketName string) (string, error) { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("policy", "") // Execute GET on bucket to list objects. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return "", err } if resp != nil { if resp.StatusCode != http.StatusOK { return "", httpRespToErrorResponse(resp, bucketName, "") } } bucketPolicyBuf, err := io.ReadAll(resp.Body) if err != nil { return "", err } policy := string(bucketPolicyBuf) return policy, err } minio-go-7.0.97/api-bucket-qos.go000066400000000000000000000136411510244170000165060ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2025 MinIO, Inc. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "encoding/json" "io" "net/http" "net/url" "strings" "github.com/minio/minio-go/v7/pkg/s3utils" "gopkg.in/yaml.v3" ) // QOSConfigVersionCurrent is the current version of the QoS configuration. const QOSConfigVersionCurrent = "v1" // QOSConfig represents the QoS configuration for a bucket. type QOSConfig struct { Version string `yaml:"version"` Rules []QOSRule `yaml:"rules"` } // QOSRule represents a single QoS rule. type QOSRule struct { ID string `yaml:"id"` Label string `yaml:"label,omitempty"` Priority int `yaml:"priority"` ObjectPrefix string `yaml:"objectPrefix"` API string `yaml:"api"` Rate int64 `yaml:"rate"` Burst int64 `yaml:"burst"` // not required for concurrency limit Limit string `yaml:"limit"` // "concurrency" or "rps" } // NewQOSConfig creates a new empty QoS configuration. func NewQOSConfig() *QOSConfig { return &QOSConfig{ Version: "v1", Rules: []QOSRule{}, } } // GetBucketQOS retrieves the Quality of Service (QoS) configuration for the bucket. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucket: Name of the bucket // // Returns the QoS configuration or an error if the operation fails. func (c *Client) GetBucketQOS(ctx context.Context, bucket string) (*QOSConfig, error) { var qosCfg QOSConfig // Input validation. if err := s3utils.CheckValidBucketName(bucket); err != nil { return nil, err } urlValues := make(url.Values) urlValues.Set("qos", "") // Execute GET on bucket to fetch qos. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucket, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, bucket, "") } b, err := io.ReadAll(resp.Body) if err != nil { return nil, err } if err = yaml.Unmarshal(b, &qosCfg); err != nil { return nil, err } return &qosCfg, nil } // SetBucketQOS sets the Quality of Service (QoS) configuration for a bucket. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucket: Name of the bucket // - qosCfg: QoS configuration to apply // // Returns an error if the operation fails. func (c *Client) SetBucketQOS(ctx context.Context, bucket string, qosCfg *QOSConfig) error { // Input validation. if err := s3utils.CheckValidBucketName(bucket); err != nil { return err } data, err := yaml.Marshal(qosCfg) if err != nil { return err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("qos", "") reqMetadata := requestMetadata{ bucketName: bucket, queryValues: urlValues, contentBody: strings.NewReader(string(data)), contentLength: int64(len(data)), } // Execute PUT to upload a new bucket QoS configuration. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp != nil { if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucket, "") } } return nil } // CounterMetric returns stats for a counter type CounterMetric struct { Last1m uint64 `json:"last1m"` Last1hr uint64 `json:"last1hr"` Total uint64 `json:"total"` } // QOSMetric - metric for a qos rule per bucket type QOSMetric struct { APIName string `json:"apiName"` Rule QOSRule `json:"rule"` Totals CounterMetric `json:"totals"` Throttled CounterMetric `json:"throttleCount"` ExceededRateLimit CounterMetric `json:"exceededRateLimitCount"` ClientDisconnCount CounterMetric `json:"clientDisconnectCount"` ReqTimeoutCount CounterMetric `json:"reqTimeoutCount"` } // QOSNodeStats represents stats for a bucket on a single node type QOSNodeStats struct { Stats []QOSMetric `json:"stats"` NodeName string `json:"node"` } // GetBucketQOSMetrics retrieves Quality of Service (QoS) metrics for a bucket. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - nodeName: Name of the node (empty string for all nodes) // // Returns QoS metrics per node or an error if the operation fails. func (c *Client) GetBucketQOSMetrics(ctx context.Context, bucketName, nodeName string) (qs []QOSNodeStats, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return qs, err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("qos-metrics", "") if nodeName != "" { urlValues.Set("node", nodeName) } // Execute GET on bucket to get qos metrics. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) defer closeResponse(resp) if err != nil { return qs, err } if resp.StatusCode != http.StatusOK { return qs, httpRespToErrorResponse(resp, bucketName, "") } respBytes, err := io.ReadAll(resp.Body) if err != nil { return qs, err } if err := json.Unmarshal(respBytes, &qs); err != nil { return qs, err } return qs, nil } minio-go-7.0.97/api-bucket-replication.go000066400000000000000000000335051510244170000202160ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/json" "encoding/xml" "io" "net/http" "net/url" "time" "github.com/google/uuid" "github.com/minio/minio-go/v7/pkg/replication" "github.com/minio/minio-go/v7/pkg/s3utils" ) // RemoveBucketReplication removes the replication configuration from an existing bucket. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // // Returns an error if the operation fails. func (c *Client) RemoveBucketReplication(ctx context.Context, bucketName string) error { return c.removeBucketReplication(ctx, bucketName) } // SetBucketReplication sets the replication configuration on an existing bucket. // If the provided configuration is empty, this method removes the existing replication configuration. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - cfg: Replication configuration to apply // // Returns an error if the operation fails. func (c *Client) SetBucketReplication(ctx context.Context, bucketName string, cfg replication.Config) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } // If replication is empty then delete it. if cfg.Empty() { return c.removeBucketReplication(ctx, bucketName) } // Save the updated replication. return c.putBucketReplication(ctx, bucketName, cfg) } // Saves a new bucket replication. func (c *Client) putBucketReplication(ctx context.Context, bucketName string, cfg replication.Config) error { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("replication", "") replication, err := xml.Marshal(cfg) if err != nil { return err } reqMetadata := requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentBody: bytes.NewReader(replication), contentLength: int64(len(replication)), contentMD5Base64: sumMD5Base64(replication), } // Execute PUT to upload a new bucket replication config. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucketName, "") } return nil } // Remove replication from a bucket. func (c *Client) removeBucketReplication(ctx context.Context, bucketName string) error { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("replication", "") // Execute DELETE on objectName. resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return err } if resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucketName, "") } return nil } // GetBucketReplication retrieves the bucket replication configuration. // If no replication configuration is found, returns an empty config with nil error. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // // Returns the replication configuration or an error if the operation fails. func (c *Client) GetBucketReplication(ctx context.Context, bucketName string) (cfg replication.Config, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return cfg, err } bucketReplicationCfg, err := c.getBucketReplication(ctx, bucketName) if err != nil { errResponse := ToErrorResponse(err) if errResponse.Code == "ReplicationConfigurationNotFoundError" { return cfg, nil } return cfg, err } return bucketReplicationCfg, nil } // Request server for current bucket replication config. func (c *Client) getBucketReplication(ctx context.Context, bucketName string) (cfg replication.Config, err error) { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("replication", "") // Execute GET on bucket to get replication config. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) defer closeResponse(resp) if err != nil { return cfg, err } if resp.StatusCode != http.StatusOK { return cfg, httpRespToErrorResponse(resp, bucketName, "") } if err = xmlDecoder(resp.Body, &cfg); err != nil { return cfg, err } return cfg, nil } // GetBucketReplicationMetrics retrieves bucket replication status metrics. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // // Returns the replication metrics or an error if the operation fails. func (c *Client) GetBucketReplicationMetrics(ctx context.Context, bucketName string) (s replication.Metrics, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return s, err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("replication-metrics", "") // Execute GET on bucket to get replication config. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) defer closeResponse(resp) if err != nil { return s, err } if resp.StatusCode != http.StatusOK { return s, httpRespToErrorResponse(resp, bucketName, "") } respBytes, err := io.ReadAll(resp.Body) if err != nil { return s, err } if err := json.Unmarshal(respBytes, &s); err != nil { return s, err } return s, nil } // mustGetUUID - get a random UUID. func mustGetUUID() string { u, err := uuid.NewRandom() if err != nil { return "" } return u.String() } // ResetBucketReplication initiates replication of previously replicated objects. // This requires ExistingObjectReplication to be enabled in the replication configuration. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - olderThan: Only replicate objects older than this duration (0 for all objects) // // Returns a reset ID that can be used to track the operation, or an error if the operation fails. func (c *Client) ResetBucketReplication(ctx context.Context, bucketName string, olderThan time.Duration) (rID string, err error) { rID = mustGetUUID() _, err = c.resetBucketReplicationOnTarget(ctx, bucketName, olderThan, "", rID) if err != nil { return rID, err } return rID, nil } // ResetBucketReplicationOnTarget initiates replication of previously replicated objects to a specific target. // This requires ExistingObjectReplication to be enabled in the replication configuration. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - olderThan: Only replicate objects older than this duration (0 for all objects) // - tgtArn: ARN of the target to reset replication for // // Returns resync target information or an error if the operation fails. func (c *Client) ResetBucketReplicationOnTarget(ctx context.Context, bucketName string, olderThan time.Duration, tgtArn string) (replication.ResyncTargetsInfo, error) { return c.resetBucketReplicationOnTarget(ctx, bucketName, olderThan, tgtArn, mustGetUUID()) } // ResetBucketReplication kicks off replication of previously replicated objects if ExistingObjectReplication // is enabled in the replication config func (c *Client) resetBucketReplicationOnTarget(ctx context.Context, bucketName string, olderThan time.Duration, tgtArn, resetID string) (rinfo replication.ResyncTargetsInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { return rinfo, err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("replication-reset", "") if olderThan > 0 { urlValues.Set("older-than", olderThan.String()) } if tgtArn != "" { urlValues.Set("arn", tgtArn) } urlValues.Set("reset-id", resetID) // Execute GET on bucket to get replication config. resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) defer closeResponse(resp) if err != nil { return rinfo, err } if resp.StatusCode != http.StatusOK { return rinfo, httpRespToErrorResponse(resp, bucketName, "") } if err = json.NewDecoder(resp.Body).Decode(&rinfo); err != nil { return rinfo, err } return rinfo, nil } // GetBucketReplicationResyncStatus retrieves the status of a replication resync operation. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - arn: ARN of the replication target (empty string for all targets) // // Returns resync status information or an error if the operation fails. func (c *Client) GetBucketReplicationResyncStatus(ctx context.Context, bucketName, arn string) (rinfo replication.ResyncTargetsInfo, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return rinfo, err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("replication-reset-status", "") if arn != "" { urlValues.Set("arn", arn) } // Execute GET on bucket to get replication config. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) defer closeResponse(resp) if err != nil { return rinfo, err } if resp.StatusCode != http.StatusOK { return rinfo, httpRespToErrorResponse(resp, bucketName, "") } if err = json.NewDecoder(resp.Body).Decode(&rinfo); err != nil { return rinfo, err } return rinfo, nil } // CancelBucketReplicationResync cancels an in-progress replication resync operation. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - tgtArn: ARN of the replication target (empty string for all targets) // // Returns the ID of the canceled resync operation or an error if the operation fails. func (c *Client) CancelBucketReplicationResync(ctx context.Context, bucketName string, tgtArn string) (id string, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { return id, err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("replication-reset-cancel", "") if tgtArn != "" { urlValues.Set("arn", tgtArn) } // Execute GET on bucket to get replication config. resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) defer closeResponse(resp) if err != nil { return id, err } if resp.StatusCode != http.StatusOK { return id, httpRespToErrorResponse(resp, bucketName, "") } strBuf, err := io.ReadAll(resp.Body) if err != nil { return "", err } id = string(strBuf) return id, nil } // GetBucketReplicationMetricsV2 retrieves bucket replication status metrics using the V2 API. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // // Returns the V2 replication metrics or an error if the operation fails. func (c *Client) GetBucketReplicationMetricsV2(ctx context.Context, bucketName string) (s replication.MetricsV2, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return s, err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("replication-metrics", "2") // Execute GET on bucket to get replication metrics. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) defer closeResponse(resp) if err != nil { return s, err } if resp.StatusCode != http.StatusOK { return s, httpRespToErrorResponse(resp, bucketName, "") } respBytes, err := io.ReadAll(resp.Body) if err != nil { return s, err } if err := json.Unmarshal(respBytes, &s); err != nil { return s, err } return s, nil } // CheckBucketReplication validates whether replication is properly configured for a bucket. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // // Returns nil if replication is valid, or an error describing the validation failure. func (c *Client) CheckBucketReplication(ctx context.Context, bucketName string) (err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("replication-check", "") // Execute GET on bucket to get replication config. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) defer closeResponse(resp) if err != nil { return err } if resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucketName, "") } return nil } minio-go-7.0.97/api-bucket-tagging.go000066400000000000000000000106261510244170000173240ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/xml" "errors" "io" "net/http" "net/url" "github.com/minio/minio-go/v7/pkg/s3utils" "github.com/minio/minio-go/v7/pkg/tags" ) // GetBucketTagging fetches the tagging configuration for a bucket. // It uses the provided context to control cancellations and timeouts. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // // Returns the bucket's tags or an error if the operation fails. func (c *Client) GetBucketTagging(ctx context.Context, bucketName string) (*tags.Tags, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("tagging", "") // Execute GET on bucket to get tagging configuration. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) defer closeResponse(resp) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, bucketName, "") } defer io.Copy(io.Discard, resp.Body) return tags.ParseBucketXML(resp.Body) } // SetBucketTagging sets the tagging configuration for a bucket. // It uses the provided context to control cancellations and timeouts. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - tags: Tag set to apply to the bucket // // Returns an error if the operation fails or if tags is nil. func (c *Client) SetBucketTagging(ctx context.Context, bucketName string, tags *tags.Tags) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } if tags == nil { return errors.New("nil tags passed") } buf, err := xml.Marshal(tags) if err != nil { return err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("tagging", "") // Content-length is mandatory to set a default encryption configuration reqMetadata := requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentBody: bytes.NewReader(buf), contentLength: int64(len(buf)), contentMD5Base64: sumMD5Base64(buf), } // Execute PUT on bucket to put tagging configuration. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { return httpRespToErrorResponse(resp, bucketName, "") } return nil } // RemoveBucketTagging removes the tagging configuration from a bucket. // It uses the provided context to control cancellations and timeouts. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // // Returns an error if the operation fails. func (c *Client) RemoveBucketTagging(ctx context.Context, bucketName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("tagging", "") // Execute DELETE on bucket to remove tagging configuration. resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return err } if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { return httpRespToErrorResponse(resp, bucketName, "") } return nil } minio-go-7.0.97/api-bucket-versioning.go000066400000000000000000000107711510244170000200700ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/xml" "net/http" "net/url" "github.com/minio/minio-go/v7/pkg/s3utils" ) // SetBucketVersioning sets a bucket versioning configuration func (c *Client) SetBucketVersioning(ctx context.Context, bucketName string, config BucketVersioningConfiguration) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } buf, err := xml.Marshal(config) if err != nil { return err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("versioning", "") reqMetadata := requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentBody: bytes.NewReader(buf), contentLength: int64(len(buf)), contentMD5Base64: sumMD5Base64(buf), contentSHA256Hex: sum256Hex(buf), } // Execute PUT to set a bucket versioning. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp != nil { if resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucketName, "") } } return nil } // EnableVersioning - enable object versioning in given bucket. func (c *Client) EnableVersioning(ctx context.Context, bucketName string) error { return c.SetBucketVersioning(ctx, bucketName, BucketVersioningConfiguration{Status: "Enabled"}) } // SuspendVersioning - suspend object versioning in given bucket. func (c *Client) SuspendVersioning(ctx context.Context, bucketName string) error { return c.SetBucketVersioning(ctx, bucketName, BucketVersioningConfiguration{Status: "Suspended"}) } // ExcludedPrefix - holds individual prefixes excluded from being versioned. type ExcludedPrefix struct { Prefix string } // BucketVersioningConfiguration is the versioning configuration structure type BucketVersioningConfiguration struct { XMLName xml.Name `xml:"VersioningConfiguration"` Status string `xml:"Status"` MFADelete string `xml:"MfaDelete,omitempty"` // MinIO extension - allows selective, prefix-level versioning exclusion. // Requires versioning to be enabled ExcludedPrefixes []ExcludedPrefix `xml:",omitempty"` ExcludeFolders bool `xml:",omitempty"` PurgeOnDelete string `xml:",omitempty"` } // Various supported states const ( Enabled = "Enabled" // Disabled State = "Disabled" only used by MFA Delete not supported yet. Suspended = "Suspended" ) // Enabled returns true if bucket versioning is enabled func (b BucketVersioningConfiguration) Enabled() bool { return b.Status == Enabled } // Suspended returns true if bucket versioning is suspended func (b BucketVersioningConfiguration) Suspended() bool { return b.Status == Suspended } // GetBucketVersioning gets the versioning configuration on // an existing bucket with a context to control cancellations and timeouts. func (c *Client) GetBucketVersioning(ctx context.Context, bucketName string) (BucketVersioningConfiguration, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return BucketVersioningConfiguration{}, err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("versioning", "") // Execute GET on bucket to get the versioning configuration. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, }) defer closeResponse(resp) if err != nil { return BucketVersioningConfiguration{}, err } if resp.StatusCode != http.StatusOK { return BucketVersioningConfiguration{}, httpRespToErrorResponse(resp, bucketName, "") } versioningConfig := BucketVersioningConfiguration{} if err = xmlDecoder(resp.Body, &versioningConfig); err != nil { return versioningConfig, err } return versioningConfig, nil } minio-go-7.0.97/api-compose-object.go000066400000000000000000000460621510244170000173450ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017, 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "fmt" "io" "net/http" "net/url" "strconv" "strings" "time" "github.com/google/uuid" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/s3utils" "github.com/minio/minio-go/v7/pkg/tags" ) // CopyDestOptions represents options specified by user for CopyObject/ComposeObject APIs type CopyDestOptions struct { Bucket string // points to destination bucket Object string // points to destination object // `Encryption` is the key info for server-side-encryption with customer // provided key. If it is nil, no encryption is performed. Encryption encrypt.ServerSide ChecksumType ChecksumType // `userMeta` is the user-metadata key-value pairs to be set on the // destination. The keys are automatically prefixed with `x-amz-meta-` // if needed. If nil is passed, and if only a single source (of any // size) is provided in the ComposeObject call, then metadata from the // source is copied to the destination. // if no user-metadata is provided, it is copied from source // (when there is only one source object in the compose // request) UserMetadata map[string]string // UserMetadata is only set to destination if ReplaceMetadata is true // other value is UserMetadata is ignored and we preserve src.UserMetadata // NOTE: if you set this value to true and now metadata is present // in UserMetadata your destination object will not have any metadata // set. ReplaceMetadata bool // `userTags` is the user defined object tags to be set on destination. // This will be set only if the `replaceTags` field is set to true. // Otherwise this field is ignored UserTags map[string]string ReplaceTags bool // Specifies whether you want to apply a Legal Hold to the copied object. LegalHold LegalHoldStatus // Object Retention related fields Mode RetentionMode RetainUntilDate time.Time Expires time.Time ContentType string ContentEncoding string ContentDisposition string ContentLanguage string CacheControl string Size int64 // Needs to be specified if progress bar is specified. // Progress of the entire copy operation will be sent here. Progress io.Reader } // Process custom-metadata to remove a `x-amz-meta-` prefix if // present and validate that keys are distinct (after this // prefix removal). func filterCustomMeta(userMeta map[string]string) map[string]string { m := make(map[string]string) for k, v := range userMeta { if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { k = k[len("x-amz-meta-"):] } if _, ok := m[k]; ok { continue } m[k] = v } return m } // Marshal converts all the CopyDestOptions into their // equivalent HTTP header representation func (opts CopyDestOptions) Marshal(header http.Header) { const replaceDirective = "REPLACE" if opts.ReplaceTags { header.Set(amzTaggingHeaderDirective, replaceDirective) if tags, _ := tags.NewTags(opts.UserTags, true); tags != nil { header.Set(amzTaggingHeader, tags.String()) } } if opts.LegalHold != LegalHoldStatus("") { header.Set(amzLegalHoldHeader, opts.LegalHold.String()) } if opts.Mode != RetentionMode("") && !opts.RetainUntilDate.IsZero() { header.Set(amzLockMode, opts.Mode.String()) header.Set(amzLockRetainUntil, opts.RetainUntilDate.Format(time.RFC3339)) } if opts.Encryption != nil { opts.Encryption.Marshal(header) } if opts.ContentType != "" { header.Set("Content-Type", opts.ContentType) } if opts.ContentEncoding != "" { header.Set("Content-Encoding", opts.ContentEncoding) } if opts.ContentDisposition != "" { header.Set("Content-Disposition", opts.ContentDisposition) } if opts.ContentLanguage != "" { header.Set("Content-Language", opts.ContentLanguage) } if opts.CacheControl != "" { header.Set("Cache-Control", opts.CacheControl) } if !opts.Expires.IsZero() { header.Set("Expires", opts.Expires.UTC().Format(http.TimeFormat)) } if opts.ChecksumType.IsSet() { header.Set(amzChecksumAlgo, opts.ChecksumType.String()) } if opts.ReplaceMetadata { header.Set("x-amz-metadata-directive", replaceDirective) for k, v := range filterCustomMeta(opts.UserMetadata) { if isAmzHeader(k) || isStandardHeader(k) || isStorageClassHeader(k) || isMinioHeader(k) { header.Set(k, v) } else { header.Set("x-amz-meta-"+k, v) } } } } // toDestinationInfo returns a validated copyOptions object. func (opts CopyDestOptions) validate() (err error) { // Input validation. if err = s3utils.CheckValidBucketName(opts.Bucket); err != nil { return err } if err = s3utils.CheckValidObjectName(opts.Object); err != nil { return err } if opts.Progress != nil && opts.Size < 0 { return errInvalidArgument("For progress bar effective size needs to be specified") } return nil } // CopySrcOptions represents a source object to be copied, using // server-side copying APIs. type CopySrcOptions struct { Bucket, Object string VersionID string MatchETag string NoMatchETag string MatchModifiedSince time.Time MatchUnmodifiedSince time.Time MatchRange bool Start, End int64 Encryption encrypt.ServerSide } // Marshal converts all the CopySrcOptions into their // equivalent HTTP header representation func (opts CopySrcOptions) Marshal(header http.Header) { // Set the source header header.Set("x-amz-copy-source", s3utils.EncodePath(opts.Bucket+"/"+opts.Object)) if opts.VersionID != "" { header.Set("x-amz-copy-source", s3utils.EncodePath(opts.Bucket+"/"+opts.Object)+"?versionId="+opts.VersionID) } if opts.MatchETag != "" { header.Set("x-amz-copy-source-if-match", opts.MatchETag) } if opts.NoMatchETag != "" { header.Set("x-amz-copy-source-if-none-match", opts.NoMatchETag) } if !opts.MatchModifiedSince.IsZero() { header.Set("x-amz-copy-source-if-modified-since", opts.MatchModifiedSince.Format(http.TimeFormat)) } if !opts.MatchUnmodifiedSince.IsZero() { header.Set("x-amz-copy-source-if-unmodified-since", opts.MatchUnmodifiedSince.Format(http.TimeFormat)) } if opts.Encryption != nil { encrypt.SSECopy(opts.Encryption).Marshal(header) } } func (opts CopySrcOptions) validate() (err error) { // Input validation. if err = s3utils.CheckValidBucketName(opts.Bucket); err != nil { return err } if err = s3utils.CheckValidObjectName(opts.Object); err != nil { return err } if opts.Start > opts.End || opts.Start < 0 { return errInvalidArgument("start must be non-negative, and start must be at most end.") } return nil } // Low level implementation of CopyObject API, supports only upto 5GiB worth of copy. func (c *Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, metadata map[string]string, srcOpts CopySrcOptions, dstOpts PutObjectOptions, ) (ObjectInfo, error) { // Build headers. headers := make(http.Header) // Set all the metadata headers. for k, v := range metadata { headers.Set(k, v) } if !dstOpts.Internal.ReplicationStatus.Empty() { headers.Set(amzBucketReplicationStatus, string(dstOpts.Internal.ReplicationStatus)) } if !dstOpts.Internal.SourceMTime.IsZero() { headers.Set(minIOBucketSourceMTime, dstOpts.Internal.SourceMTime.Format(time.RFC3339Nano)) } if dstOpts.Internal.SourceETag != "" { headers.Set(minIOBucketSourceETag, dstOpts.Internal.SourceETag) } if dstOpts.Internal.ReplicationRequest { headers.Set(minIOBucketReplicationRequest, "true") } if dstOpts.Internal.ReplicationValidityCheck { headers.Set(minIOBucketReplicationCheck, "true") } if !dstOpts.Internal.LegalholdTimestamp.IsZero() { headers.Set(minIOBucketReplicationObjectLegalHoldTimestamp, dstOpts.Internal.LegalholdTimestamp.Format(time.RFC3339Nano)) } if !dstOpts.Internal.RetentionTimestamp.IsZero() { headers.Set(minIOBucketReplicationObjectRetentionTimestamp, dstOpts.Internal.RetentionTimestamp.Format(time.RFC3339Nano)) } if !dstOpts.Internal.TaggingTimestamp.IsZero() { headers.Set(minIOBucketReplicationTaggingTimestamp, dstOpts.Internal.TaggingTimestamp.Format(time.RFC3339Nano)) } if len(dstOpts.UserTags) != 0 { if tags, _ := tags.NewTags(dstOpts.UserTags, true); tags != nil { headers.Set(amzTaggingHeader, tags.String()) } } reqMetadata := requestMetadata{ bucketName: destBucket, objectName: destObject, customHeader: headers, } if dstOpts.Internal.SourceVersionID != "" { if dstOpts.Internal.SourceVersionID != nullVersionID { if _, err := uuid.Parse(dstOpts.Internal.SourceVersionID); err != nil { return ObjectInfo{}, errInvalidArgument(err.Error()) } } urlValues := make(url.Values) urlValues.Set("versionId", dstOpts.Internal.SourceVersionID) reqMetadata.queryValues = urlValues } // Set the source header headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject)) if srcOpts.VersionID != "" { headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject)+"?versionId="+srcOpts.VersionID) } // Send upload-part-copy request resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return ObjectInfo{}, err } // Check if we got an error response. if resp.StatusCode != http.StatusOK { return ObjectInfo{}, httpRespToErrorResponse(resp, srcBucket, srcObject) } cpObjRes := copyObjectResult{} err = xmlDecoder(resp.Body, &cpObjRes) if err != nil { return ObjectInfo{}, err } objInfo := ObjectInfo{ Key: destObject, ETag: strings.Trim(cpObjRes.ETag, "\""), LastModified: cpObjRes.LastModified, } return objInfo, nil } func (c *Client) copyObjectPartDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, metadata map[string]string, ) (p CompletePart, err error) { headers := make(http.Header) // Set source headers.Set("x-amz-copy-source", s3utils.EncodePath(srcBucket+"/"+srcObject)) if startOffset < 0 { return p, errInvalidArgument("startOffset must be non-negative") } if length >= 0 { headers.Set("x-amz-copy-source-range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1)) } for k, v := range metadata { headers.Set(k, v) } queryValues := make(url.Values) queryValues.Set("partNumber", strconv.Itoa(partID)) queryValues.Set("uploadId", uploadID) resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{ bucketName: destBucket, objectName: destObject, customHeader: headers, queryValues: queryValues, }) defer closeResponse(resp) if err != nil { return p, err } // Check if we got an error response. if resp.StatusCode != http.StatusOK { return p, httpRespToErrorResponse(resp, destBucket, destObject) } // Decode copy-part response on success. cpObjRes := copyObjectResult{} err = xmlDecoder(resp.Body, &cpObjRes) if err != nil { return p, err } p.PartNumber, p.ETag = partID, cpObjRes.ETag return p, nil } // uploadPartCopy - helper function to create a part in a multipart // upload via an upload-part-copy request // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPartCopy.html func (c *Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID string, partNumber int, headers http.Header, ) (p CompletePart, err error) { // Build query parameters urlValues := make(url.Values) urlValues.Set("partNumber", strconv.Itoa(partNumber)) urlValues.Set("uploadId", uploadID) // Send upload-part-copy request resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{ bucketName: bucket, objectName: object, customHeader: headers, queryValues: urlValues, }) defer closeResponse(resp) if err != nil { return p, err } // Check if we got an error response. if resp.StatusCode != http.StatusOK { return p, httpRespToErrorResponse(resp, bucket, object) } // Decode copy-part response on success. cpObjRes := copyObjectResult{} err = xmlDecoder(resp.Body, &cpObjRes) if err != nil { return p, err } p.PartNumber, p.ETag = partNumber, cpObjRes.ETag return p, nil } // ComposeObject - creates an object using server-side copying // of existing objects. It takes a list of source objects (with optional offsets) // and concatenates them into a new object using only server-side copying // operations. Optionally takes progress reader hook for applications to // look at current progress. func (c *Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ...CopySrcOptions) (UploadInfo, error) { if len(srcs) < 1 || len(srcs) > maxPartsCount { return UploadInfo{}, errInvalidArgument("There must be as least one and up to 10000 source objects.") } for _, src := range srcs { if err := src.validate(); err != nil { return UploadInfo{}, err } } if err := dst.validate(); err != nil { return UploadInfo{}, err } srcObjectInfos := make([]ObjectInfo, len(srcs)) srcObjectSizes := make([]int64, len(srcs)) var totalSize, totalParts int64 var err error for i, src := range srcs { opts := StatObjectOptions{ServerSideEncryption: encrypt.SSE(src.Encryption), VersionID: src.VersionID} srcObjectInfos[i], err = c.StatObject(context.Background(), src.Bucket, src.Object, opts) if err != nil { return UploadInfo{}, err } srcCopySize := srcObjectInfos[i].Size // Check if a segment is specified, and if so, is the // segment within object bounds? if src.MatchRange { // Since range is specified, // 0 <= src.start <= src.end // so only invalid case to check is: if src.End >= srcCopySize || src.Start < 0 { return UploadInfo{}, errInvalidArgument( fmt.Sprintf("CopySrcOptions %d has invalid segment-to-copy [%d, %d] (size is %d)", i, src.Start, src.End, srcCopySize)) } srcCopySize = src.End - src.Start + 1 } // Only the last source may be less than `absMinPartSize` if srcCopySize < absMinPartSize && i < len(srcs)-1 { return UploadInfo{}, errInvalidArgument( fmt.Sprintf("CopySrcOptions %d is too small (%d) and it is not the last part", i, srcCopySize)) } // Is data to copy too large? totalSize += srcCopySize if totalSize > maxMultipartPutObjectSize { return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Cannot compose an object of size %d (> 5TiB)", totalSize)) } // record source size srcObjectSizes[i] = srcCopySize // calculate parts needed for current source totalParts += partsRequired(srcCopySize) // Do we need more parts than we are allowed? if totalParts > maxPartsCount { return UploadInfo{}, errInvalidArgument(fmt.Sprintf( "Your proposed compose object requires more than %d parts", maxPartsCount)) } } // Single source object case (i.e. when only one source is // involved, it is being copied wholly and at most 5GiB in // size, emptyfiles are also supported). if (totalParts == 1 && srcs[0].Start == -1 && totalSize <= maxPartSize) || (totalSize == 0) { return c.CopyObject(ctx, dst, srcs[0]) } // Now, handle multipart-copy cases. // 1. Ensure that the object has not been changed while // we are copying data. for i, src := range srcs { src.MatchETag = srcObjectInfos[i].ETag } // 2. Initiate a new multipart upload. // Set user-metadata on the destination object. If no // user-metadata is specified, and there is only one source, // (only) then metadata from source is copied. var userMeta map[string]string if dst.ReplaceMetadata { userMeta = dst.UserMetadata } else { userMeta = srcObjectInfos[0].UserMetadata } var userTags map[string]string if dst.ReplaceTags { userTags = dst.UserTags } else { userTags = srcObjectInfos[0].UserTags } uploadID, err := c.newUploadID(ctx, dst.Bucket, dst.Object, PutObjectOptions{ ServerSideEncryption: dst.Encryption, UserMetadata: userMeta, UserTags: userTags, Mode: dst.Mode, RetainUntilDate: dst.RetainUntilDate, LegalHold: dst.LegalHold, }) if err != nil { return UploadInfo{}, err } // 3. Perform copy part uploads objParts := []CompletePart{} partIndex := 1 for i, src := range srcs { h := make(http.Header) src.Marshal(h) if dst.Encryption != nil && dst.Encryption.Type() == encrypt.SSEC { dst.Encryption.Marshal(h) } // calculate start/end indices of parts after // splitting. startIdx, endIdx := calculateEvenSplits(srcObjectSizes[i], src) for j, start := range startIdx { end := endIdx[j] // Add (or reset) source range header for // upload part copy request. h.Set("x-amz-copy-source-range", fmt.Sprintf("bytes=%d-%d", start, end)) // make upload-part-copy request complPart, err := c.uploadPartCopy(ctx, dst.Bucket, dst.Object, uploadID, partIndex, h) if err != nil { return UploadInfo{}, err } if dst.Progress != nil { io.CopyN(io.Discard, dst.Progress, end-start+1) } objParts = append(objParts, complPart) partIndex++ } } // 4. Make final complete-multipart request. uploadInfo, err := c.completeMultipartUpload(ctx, dst.Bucket, dst.Object, uploadID, completeMultipartUpload{Parts: objParts}, PutObjectOptions{ServerSideEncryption: dst.Encryption}) if err != nil { return UploadInfo{}, err } uploadInfo.Size = totalSize return uploadInfo, nil } // partsRequired is maximum parts possible with // max part size of ceiling(maxMultipartPutObjectSize / (maxPartsCount - 1)) func partsRequired(size int64) int64 { maxPartSize := maxMultipartPutObjectSize / (maxPartsCount - 1) r := size / int64(maxPartSize) if size%int64(maxPartSize) > 0 { r++ } return r } // calculateEvenSplits - computes splits for a source and returns // start and end index slices. Splits happen evenly to be sure that no // part is less than 5MiB, as that could fail the multipart request if // it is not the last part. func calculateEvenSplits(size int64, src CopySrcOptions) (startIndex, endIndex []int64) { if size == 0 { return startIndex, endIndex } reqParts := partsRequired(size) startIndex = make([]int64, reqParts) endIndex = make([]int64, reqParts) // Compute number of required parts `k`, as: // // k = ceiling(size / copyPartSize) // // Now, distribute the `size` bytes in the source into // k parts as evenly as possible: // // r parts sized (q+1) bytes, and // (k - r) parts sized q bytes, where // // size = q * k + r (by simple division of size by k, // so that 0 <= r < k) // start := src.Start if start == -1 { start = 0 } quot, rem := size/reqParts, size%reqParts nextStart := start for j := int64(0); j < reqParts; j++ { curPartSize := quot if j < rem { curPartSize++ } cStart := nextStart cEnd := cStart + curPartSize - 1 nextStart = cEnd + 1 startIndex[j], endIndex[j] = cStart, cEnd } return startIndex, endIndex } minio-go-7.0.97/api-compose-object_test.go000066400000000000000000000122371510244170000204010ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "net/http" "reflect" "strings" "testing" ) const ( gb1 = 1024 * 1024 * 1024 gb5 = 5 * gb1 gb5p1 = gb5 + 1 gb10p1 = 2*gb5 + 1 gb10p2 = 2*gb5 + 2 ) func TestPartsRequired(t *testing.T) { testCases := []struct { size, ref int64 }{ {0, 0}, {1, 1}, {gb5, 10}, {gb5p1, 10}, {2 * gb5, 20}, {gb10p1, 20}, {gb10p2, 20}, {gb10p1 + gb10p2, 40}, {maxMultipartPutObjectSize, 10000}, } for i, testCase := range testCases { res := partsRequired(testCase.size) if res != testCase.ref { t.Errorf("Test %d - output did not match with reference results, Expected %d, got %d", i+1, testCase.ref, res) } } } func TestCalculateEvenSplits(t *testing.T) { testCases := []struct { // input size and source object size int64 src CopySrcOptions // output part-indexes starts, ends []int64 }{ {0, CopySrcOptions{Start: -1}, nil, nil}, {1, CopySrcOptions{Start: -1}, []int64{0}, []int64{0}}, {1, CopySrcOptions{Start: 0}, []int64{0}, []int64{0}}, {gb1, CopySrcOptions{Start: -1}, []int64{0, 536870912}, []int64{536870911, 1073741823}}, { gb5, CopySrcOptions{Start: -1}, []int64{ 0, 536870912, 1073741824, 1610612736, 2147483648, 2684354560, 3221225472, 3758096384, 4294967296, 4831838208, }, []int64{ 536870911, 1073741823, 1610612735, 2147483647, 2684354559, 3221225471, 3758096383, 4294967295, 4831838207, 5368709119, }, }, // 2 part splits { gb5p1, CopySrcOptions{Start: -1}, []int64{ 0, 536870913, 1073741825, 1610612737, 2147483649, 2684354561, 3221225473, 3758096385, 4294967297, 4831838209, }, []int64{ 536870912, 1073741824, 1610612736, 2147483648, 2684354560, 3221225472, 3758096384, 4294967296, 4831838208, 5368709120, }, }, { gb5p1, CopySrcOptions{Start: -1}, []int64{ 0, 536870913, 1073741825, 1610612737, 2147483649, 2684354561, 3221225473, 3758096385, 4294967297, 4831838209, }, []int64{ 536870912, 1073741824, 1610612736, 2147483648, 2684354560, 3221225472, 3758096384, 4294967296, 4831838208, 5368709120, }, }, // 3 part splits { gb10p1, CopySrcOptions{Start: -1}, []int64{ 0, 536870913, 1073741825, 1610612737, 2147483649, 2684354561, 3221225473, 3758096385, 4294967297, 4831838209, 5368709121, 5905580033, 6442450945, 6979321857, 7516192769, 8053063681, 8589934593, 9126805505, 9663676417, 10200547329, }, []int64{ 536870912, 1073741824, 1610612736, 2147483648, 2684354560, 3221225472, 3758096384, 4294967296, 4831838208, 5368709120, 5905580032, 6442450944, 6979321856, 7516192768, 8053063680, 8589934592, 9126805504, 9663676416, 10200547328, 10737418240, }, }, { gb10p2, CopySrcOptions{Start: -1}, []int64{ 0, 536870913, 1073741826, 1610612738, 2147483650, 2684354562, 3221225474, 3758096386, 4294967298, 4831838210, 5368709122, 5905580034, 6442450946, 6979321858, 7516192770, 8053063682, 8589934594, 9126805506, 9663676418, 10200547330, }, []int64{ 536870912, 1073741825, 1610612737, 2147483649, 2684354561, 3221225473, 3758096385, 4294967297, 4831838209, 5368709121, 5905580033, 6442450945, 6979321857, 7516192769, 8053063681, 8589934593, 9126805505, 9663676417, 10200547329, 10737418241, }, }, } for i, testCase := range testCases { resStart, resEnd := calculateEvenSplits(testCase.size, testCase.src) if !reflect.DeepEqual(testCase.starts, resStart) || !reflect.DeepEqual(testCase.ends, resEnd) { t.Errorf("Test %d - output did not match with reference results, Expected %d/%d, got %d/%d", i+1, testCase.starts, testCase.ends, resStart, resEnd) } } } func TestDestOptions(t *testing.T) { userMetadata := map[string]string{ "test": "test", "x-amz-acl": "public-read-write", "content-type": "application/binary", "X-Amz-Storage-Class": "rrs", "x-amz-grant-write": "test@exo.ch", } r := make(http.Header) dst := CopyDestOptions{ Bucket: "bucket", Object: "object", ReplaceMetadata: true, UserMetadata: userMetadata, } dst.Marshal(r) if v := r.Get("x-amz-metadata-directive"); v != "REPLACE" { t.Errorf("Test - metadata directive was expected but is missing") } for k := range r { if strings.HasSuffix(k, "test") && !strings.HasPrefix(k, "x-amz-meta-") { t.Errorf("Test meta %q was expected as an x amz meta", k) } if !strings.HasSuffix(k, "test") && strings.HasPrefix(k, "x-amz-meta-") { t.Errorf("Test an amz/standard/storageClass Header was expected but got an x amz meta data") } } } minio-go-7.0.97/api-copy-object.go000066400000000000000000000041261510244170000166450ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017, 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "io" "net/http" ) // CopyObject - copy a source object into a new object func (c *Client) CopyObject(ctx context.Context, dst CopyDestOptions, src CopySrcOptions) (UploadInfo, error) { if err := src.validate(); err != nil { return UploadInfo{}, err } if err := dst.validate(); err != nil { return UploadInfo{}, err } header := make(http.Header) dst.Marshal(header) src.Marshal(header) resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{ bucketName: dst.Bucket, objectName: dst.Object, customHeader: header, }) if err != nil { return UploadInfo{}, err } defer closeResponse(resp) if resp.StatusCode != http.StatusOK { return UploadInfo{}, httpRespToErrorResponse(resp, dst.Bucket, dst.Object) } // Update the progress properly after successful copy. if dst.Progress != nil { io.Copy(io.Discard, io.LimitReader(dst.Progress, dst.Size)) } cpObjRes := copyObjectResult{} if err = xmlDecoder(resp.Body, &cpObjRes); err != nil { return UploadInfo{}, err } // extract lifecycle expiry date and rule ID expTime, ruleID := amzExpirationToExpiryDateRuleID(resp.Header.Get(amzExpiration)) return UploadInfo{ Bucket: dst.Bucket, Key: dst.Object, LastModified: cpObjRes.LastModified, ETag: trimEtag(cpObjRes.ETag), VersionID: resp.Header.Get(amzVersionID), Expiration: expTime, ExpirationRuleID: ruleID, }, nil } minio-go-7.0.97/api-datatypes.go000066400000000000000000000166561510244170000164400ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "encoding/xml" "io" "net/http" "net/url" "strings" "time" ) // BucketInfo container for bucket metadata. type BucketInfo struct { // The name of the bucket. Name string `json:"name"` // Date the bucket was created. CreationDate time.Time `json:"creationDate"` // BucketRegion region where the bucket is present BucketRegion string `json:"bucketRegion"` } // StringMap represents map with custom UnmarshalXML type StringMap map[string]string // UnmarshalXML unmarshals the XML into a map of string to strings, // creating a key in the map for each tag and setting it's value to the // tags contents. // // The fact this function is on the pointer of Map is important, so that // if m is nil it can be initialized, which is often the case if m is // nested in another xml structural. This is also why the first thing done // on the first line is initialize it. func (m *StringMap) UnmarshalXML(d *xml.Decoder, _ xml.StartElement) error { *m = StringMap{} for { // Format is value var e struct { XMLName xml.Name Value string `xml:",chardata"` } err := d.Decode(&e) if err == io.EOF { break } if err != nil { return err } (*m)[e.XMLName.Local] = e.Value } return nil } // URLMap represents map with custom UnmarshalXML type URLMap map[string]string // UnmarshalXML unmarshals the XML into a map of string to strings, // creating a key in the map for each tag and setting it's value to the // tags contents. // // The fact this function is on the pointer of Map is important, so that // if m is nil it can be initialized, which is often the case if m is // nested in another xml structural. This is also why the first thing done // on the first line is initialize it. func (m *URLMap) UnmarshalXML(d *xml.Decoder, se xml.StartElement) error { *m = URLMap{} var tgs string if err := d.DecodeElement(&tgs, &se); err != nil { if err == io.EOF { return nil } return err } for tgs != "" { var key string key, tgs, _ = stringsCut(tgs, "&") if key == "" { continue } key, value, _ := stringsCut(key, "=") key, err := url.QueryUnescape(key) if err != nil { return err } value, err = url.QueryUnescape(value) if err != nil { return err } (*m)[key] = value } return nil } // stringsCut slices s around the first instance of sep, // returning the text before and after sep. // The found result reports whether sep appears in s. // If sep does not appear in s, cut returns s, "", false. func stringsCut(s, sep string) (before, after string, found bool) { if i := strings.Index(s, sep); i >= 0 { return s[:i], s[i+len(sep):], true } return s, "", false } // Owner name. type Owner struct { XMLName xml.Name `xml:"Owner" json:"owner"` DisplayName string `xml:"ID" json:"name"` ID string `xml:"DisplayName" json:"id"` } // UploadInfo contains information about the // newly uploaded or copied object. type UploadInfo struct { Bucket string Key string ETag string Size int64 LastModified time.Time Location string VersionID string // Lifecycle expiry-date and ruleID associated with the expiry // not to be confused with `Expires` HTTP header. Expiration time.Time ExpirationRuleID string // Verified checksum values, if any. // Values are base64 (standard) encoded. // For multipart objects this is a checksum of the checksum of each part. ChecksumCRC32 string ChecksumCRC32C string ChecksumSHA1 string ChecksumSHA256 string ChecksumCRC64NVME string ChecksumMode string } // RestoreInfo contains information of the restore operation of an archived object type RestoreInfo struct { // Is the restoring operation is still ongoing OngoingRestore bool // When the restored copy of the archived object will be removed ExpiryTime time.Time } // ObjectInfo container for object metadata. type ObjectInfo struct { // An ETag is optionally set to md5sum of an object. In case of multipart objects, // ETag is of the form MD5SUM-N where MD5SUM is md5sum of all individual md5sums of // each parts concatenated into one string. ETag string `json:"etag"` Key string `json:"name"` // Name of the object LastModified time.Time `json:"lastModified"` // Date and time the object was last modified. Size int64 `json:"size"` // Size in bytes of the object. ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data. Expires time.Time `json:"expires"` // The date and time at which the object is no longer able to be cached. // Collection of additional metadata on the object. // eg: x-amz-meta-*, content-encoding etc. Metadata http.Header `json:"metadata" xml:"-"` // x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value. // Only returned by MinIO servers. UserMetadata StringMap `json:"userMetadata,omitempty"` // x-amz-tagging values in their k/v values. // Only returned by MinIO servers. UserTags URLMap `json:"userTags,omitempty" xml:"UserTags"` // x-amz-tagging-count value UserTagCount int // Owner name. Owner Owner // ACL grant. Grant []Grant // The class of storage used to store the object. StorageClass string `json:"storageClass"` // Versioning related information IsLatest bool IsDeleteMarker bool VersionID string `xml:"VersionId"` // x-amz-replication-status value is either in one of the following states // - COMPLETED // - PENDING // - FAILED // - REPLICA (on the destination) ReplicationStatus string `xml:"ReplicationStatus"` // set to true if delete marker has backing object version on target, and eligible to replicate ReplicationReady bool // Lifecycle expiry-date and ruleID associated with the expiry // not to be confused with `Expires` HTTP header. Expiration time.Time ExpirationRuleID string // NumVersions is the number of versions of the object. NumVersions int Restore *RestoreInfo // Checksum values ChecksumCRC32 string ChecksumCRC32C string ChecksumSHA1 string ChecksumSHA256 string ChecksumCRC64NVME string ChecksumMode string Internal *struct { K int // Data blocks M int // Parity blocks } `xml:"Internal"` // Error Err error `json:"-"` } // ObjectMultipartInfo container for multipart object metadata. type ObjectMultipartInfo struct { // Date and time at which the multipart upload was initiated. Initiated time.Time `type:"timestamp" timestampFormat:"iso8601"` Initiator initiator Owner owner // The type of storage to use for the object. Defaults to 'STANDARD'. StorageClass string // Key of the object for which the multipart upload was initiated. Key string // Size in bytes of the object. Size int64 // Upload ID that identifies the multipart upload. UploadID string `xml:"UploadId"` // Error Err error } minio-go-7.0.97/api-error-response.go000066400000000000000000000204341510244170000174140ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "encoding/xml" "errors" "fmt" "io" "net/http" "strings" ) /* **** SAMPLE ERROR RESPONSE **** AccessDenied Access Denied bucketName objectName F19772218238A85A GuWkjyviSiGHizehqpmsD1ndz5NClSP19DOT+s2mv7gXGQ8/X1lhbDGiIJEXpGFD */ // ErrorResponse - Is the typed error returned by all API operations. // ErrorResponse struct should be comparable since it is compared inside // golang http API (https://github.com/golang/go/issues/29768) type ErrorResponse struct { XMLName xml.Name `xml:"Error" json:"-"` Code string Message string BucketName string Key string Resource string RequestID string `xml:"RequestId"` HostID string `xml:"HostId"` // Region where the bucket is located. This header is returned // only in HEAD bucket and ListObjects response. Region string // Captures the server string returned in response header. Server string // Underlying HTTP status code for the returned error StatusCode int `xml:"-" json:"-"` } // ToErrorResponse - Returns parsed ErrorResponse struct from body and // http headers. // // For example: // // import s3 "github.com/minio/minio-go/v7" // ... // ... // reader, stat, err := s3.GetObject(...) // if err != nil { // resp := s3.ToErrorResponse(err) // } // ... func ToErrorResponse(err error) ErrorResponse { switch err := err.(type) { case ErrorResponse: return err default: return ErrorResponse{} } } // Error - Returns S3 error string. func (e ErrorResponse) Error() string { if e.Message == "" { msg, ok := s3ErrorResponseMap[e.Code] if !ok { msg = fmt.Sprintf("Error response code %s.", e.Code) } return msg } return e.Message } // Common string for errors to report issue location in unexpected // cases. const ( reportIssue = "Please report this issue at https://github.com/minio/minio-go/issues." ) // xmlDecodeAndBody reads the whole body up to 1MB and // tries to XML decode it into v. // The body that was read and any error from reading or decoding is returned. func xmlDecodeAndBody(bodyReader io.Reader, v interface{}) ([]byte, error) { // read the whole body (up to 1MB) const maxBodyLength = 1 << 20 body, err := io.ReadAll(io.LimitReader(bodyReader, maxBodyLength)) if err != nil { return nil, err } return bytes.TrimSpace(body), xmlDecoder(bytes.NewReader(body), v) } // httpRespToErrorResponse returns a new encoded ErrorResponse // structure as error. func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) error { if resp == nil { msg := "Empty http response. " + reportIssue return errInvalidArgument(msg) } errResp := ErrorResponse{ StatusCode: resp.StatusCode, Server: resp.Header.Get("Server"), } _, success := successStatus[resp.StatusCode] errBody, err := xmlDecodeAndBody(resp.Body, &errResp) // Xml decoding failed with no body, fall back to HTTP headers. if err != nil { var unmarshalErr xml.UnmarshalError if success && errors.As(err, &unmarshalErr) { // This is a successful message so not an error response // return nil, return nil } switch resp.StatusCode { case http.StatusNotFound: if objectName == "" { errResp = ErrorResponse{ StatusCode: resp.StatusCode, Code: NoSuchBucket, Message: s3ErrorResponseMap[NoSuchBucket], BucketName: bucketName, } } else { errResp = ErrorResponse{ StatusCode: resp.StatusCode, Code: NoSuchKey, Message: s3ErrorResponseMap[NoSuchKey], BucketName: bucketName, Key: objectName, } } case http.StatusForbidden: errResp = ErrorResponse{ StatusCode: resp.StatusCode, Code: AccessDenied, Message: s3ErrorResponseMap[AccessDenied], BucketName: bucketName, Key: objectName, } case http.StatusConflict: errResp = ErrorResponse{ StatusCode: resp.StatusCode, Code: Conflict, Message: s3ErrorResponseMap[Conflict], BucketName: bucketName, } case http.StatusPreconditionFailed: errResp = ErrorResponse{ StatusCode: resp.StatusCode, Code: PreconditionFailed, Message: s3ErrorResponseMap[PreconditionFailed], BucketName: bucketName, Key: objectName, } default: msg := resp.Status if len(errBody) > 0 { msg = string(errBody) if len(msg) > 1024 { msg = msg[:1024] + "..." } } errResp = ErrorResponse{ StatusCode: resp.StatusCode, Code: resp.Status, Message: msg, BucketName: bucketName, } } } code := resp.Header.Get("x-minio-error-code") if code != "" { errResp.Code = code } desc := resp.Header.Get("x-minio-error-desc") if desc != "" { errResp.Message = strings.Trim(desc, `"`) } // Save hostID, requestID and region information // from headers if not available through error XML. if errResp.RequestID == "" { errResp.RequestID = resp.Header.Get("x-amz-request-id") } if errResp.HostID == "" { errResp.HostID = resp.Header.Get("x-amz-id-2") } if errResp.Region == "" { errResp.Region = resp.Header.Get("x-amz-bucket-region") } if errResp.Code == InvalidRegion && errResp.Region != "" { errResp.Message = fmt.Sprintf("Region does not match, expecting region ‘%s’.", errResp.Region) } return errResp } // errTransferAccelerationBucket - bucket name is invalid to be used with transfer acceleration. func errTransferAccelerationBucket(bucketName string) error { msg := "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods ‘.’." return ErrorResponse{ StatusCode: http.StatusBadRequest, Code: InvalidArgument, Message: msg, BucketName: bucketName, } } // errEntityTooLarge - Input size is larger than supported maximum. func errEntityTooLarge(totalSize, maxObjectSize int64, bucketName, objectName string) error { msg := fmt.Sprintf("Your proposed upload size ‘%d’ exceeds the maximum allowed object size ‘%d’ for single PUT operation.", totalSize, maxObjectSize) return ErrorResponse{ StatusCode: http.StatusBadRequest, Code: EntityTooLarge, Message: msg, BucketName: bucketName, Key: objectName, } } // errEntityTooSmall - Input size is smaller than supported minimum. func errEntityTooSmall(totalSize int64, bucketName, objectName string) error { msg := fmt.Sprintf("Your proposed upload size ‘%d’ is below the minimum allowed object size ‘0B’ for single PUT operation.", totalSize) return ErrorResponse{ StatusCode: http.StatusBadRequest, Code: EntityTooSmall, Message: msg, BucketName: bucketName, Key: objectName, } } // errUnexpectedEOF - Unexpected end of file reached. func errUnexpectedEOF(totalRead, totalSize int64, bucketName, objectName string) error { msg := fmt.Sprintf("Data read ‘%d’ is not equal to the size ‘%d’ of the input Reader.", totalRead, totalSize) return ErrorResponse{ StatusCode: http.StatusBadRequest, Code: UnexpectedEOF, Message: msg, BucketName: bucketName, Key: objectName, } } // errInvalidArgument - Invalid argument response. func errInvalidArgument(message string) error { return ErrorResponse{ StatusCode: http.StatusBadRequest, Code: InvalidArgument, Message: message, RequestID: "minio", } } // errAPINotSupported - API not supported response // The specified API call is not supported func errAPINotSupported(message string) error { return ErrorResponse{ StatusCode: http.StatusNotImplemented, Code: APINotSupported, Message: message, RequestID: "minio", } } minio-go-7.0.97/api-error-response_test.go000066400000000000000000000251531510244170000204560ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "encoding/xml" "fmt" "io" "net/http" "reflect" "strconv" "testing" ) // Tests validate the Error generator function for http response with error. func TestHttpRespToErrorResponse(t *testing.T) { // 'genAPIErrorResponse' generates ErrorResponse for given APIError. // provides a encodable populated response values. genAPIErrorResponse := func(err APIError, bucketName string) ErrorResponse { return ErrorResponse{ Code: err.Code, Message: err.Description, BucketName: bucketName, } } // Encodes the response headers into XML format. encodeErr := func(response ErrorResponse) []byte { buf := &bytes.Buffer{} buf.WriteString(xml.Header) encoder := xml.NewEncoder(buf) err := encoder.Encode(response) if err != nil { t.Fatalf("error encoding response: %v", err) } return buf.Bytes() } // `createErrorResponse` Mocks a generic error response from the server. createErrorResponse := func(statusCode int, body []byte) *http.Response { resp := &http.Response{} resp.StatusCode = statusCode resp.Status = http.StatusText(statusCode) resp.Body = io.NopCloser(bytes.NewBuffer(body)) return resp } // `createAPIErrorResponse` Mocks XML error response from the server. createAPIErrorResponse := func(APIErr APIError, bucketName string) *http.Response { // generate error response. // response body contains the XML error message. errorResponse := genAPIErrorResponse(APIErr, bucketName) encodedErrorResponse := encodeErr(errorResponse) return createErrorResponse(APIErr.HTTPStatusCode, encodedErrorResponse) } // 'genErrResponse' contructs error response based http Status Code genErrResponse := func(resp *http.Response, code, message, bucketName, objectName string) ErrorResponse { errResp := ErrorResponse{ StatusCode: resp.StatusCode, Code: code, Message: message, BucketName: bucketName, Key: objectName, RequestID: resp.Header.Get("x-amz-request-id"), HostID: resp.Header.Get("x-amz-id-2"), Region: resp.Header.Get("x-amz-bucket-region"), } return errResp } // Generate invalid argument error. genInvalidError := func(message string) error { errResp := ErrorResponse{ StatusCode: http.StatusBadRequest, Code: InvalidArgument, Message: message, RequestID: "minio", } return errResp } // Set common http response headers. setCommonHeaders := func(resp *http.Response) *http.Response { // set headers. resp.Header = make(http.Header) resp.Header.Set("x-amz-request-id", "xyz") resp.Header.Set("x-amz-id-2", "abc") resp.Header.Set("x-amz-bucket-region", "us-east-1") return resp } // Generate http response with empty body. // Set the StatusCode to the argument supplied. // Sets common headers. genEmptyBodyResponse := func(statusCode int) *http.Response { resp := &http.Response{ StatusCode: statusCode, Status: http.StatusText(statusCode), Body: io.NopCloser(bytes.NewReader(nil)), } setCommonHeaders(resp) return resp } // Decode XML error message from the http response body. decodeXMLError := func(resp *http.Response) error { errResp := ErrorResponse{ StatusCode: resp.StatusCode, } err := xmlDecoder(resp.Body, &errResp) if err != nil { t.Fatalf("XML decoding of response body failed: %v", err) } return errResp } // List of APIErrors used to generate/mock server side XML error response. APIErrors := []APIError{ { Code: NoSuchBucketPolicy, Description: "The specified bucket does not have a bucket policy.", HTTPStatusCode: http.StatusNotFound, }, } // List of expected response. // Used for asserting the actual response. expectedErrResponse := []error{ genInvalidError("Empty http response. " + "Please report this issue at https://github.com/minio/minio-go/issues."), decodeXMLError(createAPIErrorResponse(APIErrors[0], "minio-bucket")), genErrResponse(setCommonHeaders(&http.Response{StatusCode: http.StatusNotFound}), NoSuchBucket, s3ErrorResponseMap[NoSuchBucket], "minio-bucket", ""), genErrResponse(setCommonHeaders(&http.Response{StatusCode: http.StatusNotFound}), NoSuchKey, s3ErrorResponseMap[NoSuchKey], "minio-bucket", "Asia/"), genErrResponse(setCommonHeaders(&http.Response{StatusCode: http.StatusForbidden}), AccessDenied, s3ErrorResponseMap[AccessDenied], "minio-bucket", ""), genErrResponse(setCommonHeaders(&http.Response{StatusCode: http.StatusConflict}), Conflict, s3ErrorResponseMap[Conflict], "minio-bucket", ""), genErrResponse(setCommonHeaders(&http.Response{StatusCode: http.StatusBadRequest}), "Bad Request", "Bad Request", "minio-bucket", ""), genErrResponse(setCommonHeaders(&http.Response{StatusCode: http.StatusInternalServerError}), "Internal Server Error", "my custom object store error", "minio-bucket", ""), genErrResponse(setCommonHeaders(&http.Response{StatusCode: http.StatusInternalServerError}), "Internal Server Error", "my custom object store error, with way too long body", "minio-bucket", ""), } // List of http response to be used as input. inputResponses := []*http.Response{ nil, createAPIErrorResponse(APIErrors[0], "minio-bucket"), genEmptyBodyResponse(http.StatusNotFound), genEmptyBodyResponse(http.StatusNotFound), genEmptyBodyResponse(http.StatusForbidden), genEmptyBodyResponse(http.StatusConflict), genEmptyBodyResponse(http.StatusBadRequest), setCommonHeaders(createErrorResponse(http.StatusInternalServerError, []byte("my custom object store error\n"))), setCommonHeaders(createErrorResponse(http.StatusInternalServerError, append([]byte("my custom object store error, with way too long body\n"), bytes.Repeat([]byte("\n"), 2*1024*1024)...))), } testCases := []struct { bucketName string objectName string inputHTTPResp *http.Response // expected results. expectedResult error // flag indicating whether tests should pass. }{ {"minio-bucket", "", inputResponses[0], expectedErrResponse[0]}, {"minio-bucket", "", inputResponses[1], expectedErrResponse[1]}, {"minio-bucket", "", inputResponses[2], expectedErrResponse[2]}, {"minio-bucket", "Asia/", inputResponses[3], expectedErrResponse[3]}, {"minio-bucket", "", inputResponses[4], expectedErrResponse[4]}, {"minio-bucket", "", inputResponses[5], expectedErrResponse[5]}, {"minio-bucket", "", inputResponses[6], expectedErrResponse[6]}, {"minio-bucket", "", inputResponses[7], expectedErrResponse[7]}, {"minio-bucket", "", inputResponses[8], expectedErrResponse[8]}, } for i, testCase := range testCases { actualResult := httpRespToErrorResponse(testCase.inputHTTPResp, testCase.bucketName, testCase.objectName) if !reflect.DeepEqual(testCase.expectedResult, actualResult) { t.Errorf("Test %d: Expected result to be '%#v', but instead got '%#v'", i+1, testCase.expectedResult, actualResult) } } } // Test validates 'ErrEntityTooLarge' error response. func TestErrEntityTooLarge(t *testing.T) { msg := fmt.Sprintf("Your proposed upload size ‘%d’ exceeds the maximum allowed object size ‘%d’ for single PUT operation.", 1000000, 99999) expectedResult := ErrorResponse{ StatusCode: http.StatusBadRequest, Code: EntityTooLarge, Message: msg, BucketName: "minio-bucket", Key: "Asia/", } actualResult := errEntityTooLarge(1000000, 99999, "minio-bucket", "Asia/") if !reflect.DeepEqual(expectedResult, actualResult) { t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) } } // Test validates 'ErrEntityTooSmall' error response. func TestErrEntityTooSmall(t *testing.T) { msg := fmt.Sprintf("Your proposed upload size ‘%d’ is below the minimum allowed object size ‘0B’ for single PUT operation.", -1) expectedResult := ErrorResponse{ StatusCode: http.StatusBadRequest, Code: EntityTooSmall, Message: msg, BucketName: "minio-bucket", Key: "Asia/", } actualResult := errEntityTooSmall(-1, "minio-bucket", "Asia/") if !reflect.DeepEqual(expectedResult, actualResult) { t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) } } // Test validates 'ErrUnexpectedEOF' error response. func TestErrUnexpectedEOF(t *testing.T) { msg := fmt.Sprintf("Data read ‘%s’ is not equal to the size ‘%s’ of the input Reader.", strconv.FormatInt(100, 10), strconv.FormatInt(101, 10)) expectedResult := ErrorResponse{ StatusCode: http.StatusBadRequest, Code: UnexpectedEOF, Message: msg, BucketName: "minio-bucket", Key: "Asia/", } actualResult := errUnexpectedEOF(100, 101, "minio-bucket", "Asia/") if !reflect.DeepEqual(expectedResult, actualResult) { t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) } } // Test validates 'errInvalidArgument' response. func TestErrInvalidArgument(t *testing.T) { expectedResult := ErrorResponse{ StatusCode: http.StatusBadRequest, Code: InvalidArgument, Message: "Invalid Argument", RequestID: "minio", } actualResult := errInvalidArgument("Invalid Argument") if !reflect.DeepEqual(expectedResult, actualResult) { t.Errorf("Expected result to be '%#v', but instead got '%#v'", expectedResult, actualResult) } } // Tests if the Message field is missing. func TestErrWithoutMessage(t *testing.T) { errResp := ErrorResponse{ Code: AccessDenied, RequestID: "minio", } if errResp.Error() != s3ErrorResponseMap[AccessDenied] { t.Errorf("Expected \"%s\", got %s", s3ErrorResponseMap[AccessDenied], errResp) } errResp = ErrorResponse{ Code: InvalidArgument, RequestID: "minio", } if errResp.Error() != fmt.Sprintf("Error response code %s.", errResp.Code) { t.Errorf("Expected \"Error response code %s.\", got \"%s\"", InvalidArgument, errResp) } } // Tests if ErrorResponse is comparable since it is compared // inside golang http code (https://github.com/golang/go/issues/29768) func TestErrorResponseComparable(t *testing.T) { var e1 interface{} = ErrorResponse{} var e2 interface{} = ErrorResponse{} if e1 != e2 { t.Fatalf("ErrorResponse should be comparable") } } minio-go-7.0.97/api-get-object-acl.go000066400000000000000000000102571510244170000172110ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "encoding/xml" "net/http" "net/url" ) // Grantee represents the person being granted permissions. type Grantee struct { XMLName xml.Name `xml:"Grantee"` ID string `xml:"ID"` DisplayName string `xml:"DisplayName"` URI string `xml:"URI"` } // Grant holds grant information type Grant struct { XMLName xml.Name `xml:"Grant"` Grantee Grantee Permission string `xml:"Permission"` } // AccessControlList contains the set of grantees and the permissions assigned to each grantee. type AccessControlList struct { XMLName xml.Name `xml:"AccessControlList"` Grant []Grant Permission string `xml:"Permission"` } type accessControlPolicy struct { XMLName xml.Name `xml:"AccessControlPolicy"` Owner Owner AccessControlList AccessControlList } // GetObjectACL get object ACLs func (c *Client) GetObjectACL(ctx context.Context, bucketName, objectName string) (*ObjectInfo, error) { resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: url.Values{ "acl": []string{""}, }, }) if err != nil { return nil, err } defer closeResponse(resp) if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, bucketName, objectName) } res := &accessControlPolicy{} if err := xmlDecoder(resp.Body, res); err != nil { return nil, err } objInfo, err := c.StatObject(ctx, bucketName, objectName, StatObjectOptions{}) if err != nil { return nil, err } objInfo.Owner.DisplayName = res.Owner.DisplayName objInfo.Owner.ID = res.Owner.ID objInfo.Grant = append(objInfo.Grant, res.AccessControlList.Grant...) cannedACL := getCannedACL(res) if cannedACL != "" { objInfo.Metadata.Add("X-Amz-Acl", cannedACL) return &objInfo, nil } grantACL := getAmzGrantACL(res) for k, v := range grantACL { objInfo.Metadata[k] = v } return &objInfo, nil } func getCannedACL(aCPolicy *accessControlPolicy) string { grants := aCPolicy.AccessControlList.Grant switch { case len(grants) == 1: if grants[0].Grantee.URI == "" && grants[0].Permission == "FULL_CONTROL" { return "private" } case len(grants) == 2: for _, g := range grants { if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && g.Permission == "READ" { return "authenticated-read" } if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "READ" { return "public-read" } if g.Permission == "READ" && g.Grantee.ID == aCPolicy.Owner.ID { return "bucket-owner-read" } } case len(grants) == 3: for _, g := range grants { if g.Grantee.URI == "http://acs.amazonaws.com/groups/global/AllUsers" && g.Permission == "WRITE" { return "public-read-write" } } } return "" } func getAmzGrantACL(aCPolicy *accessControlPolicy) map[string][]string { grants := aCPolicy.AccessControlList.Grant res := map[string][]string{} for _, g := range grants { switch g.Permission { case "READ": res["X-Amz-Grant-Read"] = append(res["X-Amz-Grant-Read"], "id="+g.Grantee.ID) case "WRITE": res["X-Amz-Grant-Write"] = append(res["X-Amz-Grant-Write"], "id="+g.Grantee.ID) case "READ_ACP": res["X-Amz-Grant-Read-Acp"] = append(res["X-Amz-Grant-Read-Acp"], "id="+g.Grantee.ID) case "WRITE_ACP": res["X-Amz-Grant-Write-Acp"] = append(res["X-Amz-Grant-Write-Acp"], "id="+g.Grantee.ID) case "FULL_CONTROL": res["X-Amz-Grant-Full-Control"] = append(res["X-Amz-Grant-Full-Control"], "id="+g.Grantee.ID) } } return res } minio-go-7.0.97/api-get-object-attributes.go000066400000000000000000000127751510244170000206470ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "encoding/xml" "errors" "net/http" "net/url" "strconv" "time" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/s3utils" ) // ObjectAttributesOptions are options used for the GetObjectAttributes API // // - MaxParts // How many parts the caller wants to be returned (default: 1000) // // - VersionID // The object version you want to attributes for // // - PartNumberMarker // the listing will start AFTER the part matching PartNumberMarker // // - ServerSideEncryption // The server-side encryption algorithm used when storing this object in Minio type ObjectAttributesOptions struct { MaxParts int VersionID string PartNumberMarker int ServerSideEncryption encrypt.ServerSide } // ObjectAttributes is the response object returned by the GetObjectAttributes API // // - VersionID // The object version // // - LastModified // The last time the object was modified // // - ObjectAttributesResponse // Contains more information about the object type ObjectAttributes struct { VersionID string LastModified time.Time ObjectAttributesResponse } // ObjectAttributesResponse contains details returned by the GetObjectAttributes API // // Noteworthy fields: // // - ObjectParts.PartsCount // Contains the total part count for the object (not the current response) // // - ObjectParts.PartNumberMarker // Pagination of parts will begin at (but not include) PartNumberMarker // // - ObjectParts.NextPartNumberMarket // The next PartNumberMarker to be used in order to continue pagination // // - ObjectParts.IsTruncated // Indicates if the last part is included in the request (does not check if parts are missing from the start of the list, ONLY the end) // // - ObjectParts.MaxParts // Reflects the MaxParts used by the caller or the default MaxParts value of the API type ObjectAttributesResponse struct { ETag string `xml:",omitempty"` StorageClass string ObjectSize int Checksum struct { ChecksumCRC32 string `xml:",omitempty"` ChecksumCRC32C string `xml:",omitempty"` ChecksumSHA1 string `xml:",omitempty"` ChecksumSHA256 string `xml:",omitempty"` } ObjectParts struct { PartsCount int PartNumberMarker int NextPartNumberMarker int MaxParts int IsTruncated bool Parts []*ObjectAttributePart `xml:"Part"` } } // ObjectAttributePart is used by ObjectAttributesResponse to describe an object part type ObjectAttributePart struct { ChecksumCRC32 string `xml:",omitempty"` ChecksumCRC32C string `xml:",omitempty"` ChecksumSHA1 string `xml:",omitempty"` ChecksumSHA256 string `xml:",omitempty"` PartNumber int Size int } func (o *ObjectAttributes) parseResponse(resp *http.Response) (err error) { mod, err := parseRFC7231Time(resp.Header.Get("Last-Modified")) if err != nil { return err } o.LastModified = mod o.VersionID = resp.Header.Get(amzVersionID) response := new(ObjectAttributesResponse) if err := xml.NewDecoder(resp.Body).Decode(response); err != nil { return err } o.ObjectAttributesResponse = *response return err } // GetObjectAttributes API combines HeadObject and ListParts. // More details on usage can be found in the documentation for ObjectAttributesOptions{} func (c *Client) GetObjectAttributes(ctx context.Context, bucketName, objectName string, opts ObjectAttributesOptions) (*ObjectAttributes, error) { if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } urlValues := make(url.Values) urlValues.Add("attributes", "") if opts.VersionID != "" { urlValues.Add("versionId", opts.VersionID) } headers := make(http.Header) headers.Set(amzObjectAttributes, GetObjectAttributesTags) if opts.PartNumberMarker > 0 { headers.Set(amzPartNumberMarker, strconv.Itoa(opts.PartNumberMarker)) } if opts.MaxParts > 0 { headers.Set(amzMaxParts, strconv.Itoa(opts.MaxParts)) } else { headers.Set(amzMaxParts, strconv.Itoa(GetObjectAttributesMaxParts)) } if opts.ServerSideEncryption != nil { opts.ServerSideEncryption.Marshal(headers) } resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, customHeader: headers, }) if err != nil { return nil, err } defer closeResponse(resp) hasEtag := resp.Header.Get(ETag) if hasEtag != "" { return nil, errors.New("getObjectAttributes is not supported by the current endpoint version") } if resp.StatusCode != http.StatusOK { ER := new(ErrorResponse) if err := xml.NewDecoder(resp.Body).Decode(ER); err != nil { return nil, err } return nil, *ER } OA := new(ObjectAttributes) err = OA.parseResponse(resp) if err != nil { return nil, err } return OA, nil } minio-go-7.0.97/api-get-object-file.go000066400000000000000000000066641510244170000174000ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "io" "os" "path/filepath" "github.com/minio/minio-go/v7/pkg/s3utils" ) // FGetObject - download contents of an object to a local file. // The options can be used to specify the GET request further. func (c *Client) FGetObject(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return err } // Verify if destination already exists. st, err := os.Stat(filePath) if err == nil { // If the destination exists and is a directory. if st.IsDir() { return errInvalidArgument("fileName is a directory.") } } // Proceed if file does not exist. return for all other errors. if err != nil { if !os.IsNotExist(err) { return err } } // Extract top level directory. objectDir, _ := filepath.Split(filePath) if objectDir != "" { // Create any missing top level directories. if err := os.MkdirAll(objectDir, 0o700); err != nil { return err } } // Gather md5sum. objectStat, err := c.StatObject(ctx, bucketName, objectName, StatObjectOptions(opts)) if err != nil { return err } // Write to a temporary file "fileName.part.minio" before saving. filePartPath := filepath.Join(filepath.Dir(filePath), sum256Hex([]byte(filepath.Base(filePath)+objectStat.ETag))+".part.minio") // If exists, open in append mode. If not create it as a part file. filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) if err != nil { return err } // If we return early with an error, be sure to close and delete // filePart. If we have an error along the way there is a chance // that filePart is somehow damaged, and we should discard it. closeAndRemove := true defer func() { if closeAndRemove { _ = filePart.Close() _ = os.Remove(filePartPath) } }() // Issue Stat to get the current offset. st, err = filePart.Stat() if err != nil { return err } // Initialize get object request headers to set the // appropriate range offsets to read from. if st.Size() > 0 { opts.SetRange(st.Size(), 0) } // Seek to current position for incoming reader. objectReader, objectStat, _, err := c.getObject(ctx, bucketName, objectName, opts) if err != nil { return err } // Write to the part file. if _, err = io.CopyN(filePart, objectReader, objectStat.Size); err != nil { return err } // Close the file before rename, this is specifically needed for Windows users. closeAndRemove = false if err = filePart.Close(); err != nil { return err } // Safely completed. Now commit by renaming to actual filename. if err = os.Rename(filePartPath, filePath); err != nil { return err } // Return. return nil } minio-go-7.0.97/api-get-object-file_test.go000066400000000000000000000052641510244170000204320ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2025 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "testing" ) func TestFGetObjectReturnSuccess(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") w.Header().Set("Content-Length", "5") w.Header().Set("Etag", "abc123") w.Write([]byte("12345")) })) defer srv.Close() // New - instantiate minio client with options clnt, err := New(srv.Listener.Addr().String(), &Options{ Region: "us-east-1", }) if err != nil { t.Fatal(err) } localFilePath := filepath.Join(t.TempDir(), "minio_test_fgetobject_file") err = clnt.FGetObject(context.Background(), "bucketName", "objectName", localFilePath, GetObjectOptions{}) if err != nil { t.Fatal(err) } buf, err := os.ReadFile(localFilePath) if err != nil { t.Fatalf("Expected 'nil', got %v", err) } if len(buf) != 5 { t.Fatalf("Expected read bytes '5', got %v", len(buf)) } } func TestFGetObjectReturnSuccessIfFileNameLengthIs255(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") w.Header().Set("Content-Length", "5") w.Header().Set("Etag", "abc123") w.Write([]byte("12345")) })) defer srv.Close() // New - instantiate minio client with options clnt, err := New(srv.Listener.Addr().String(), &Options{ Region: "us-east-1", }) if err != nil { t.Fatal(err) } localFilePath := filepath.Join(t.TempDir(), strings.Repeat("a", 255)) if len(filepath.Base(localFilePath)) != 255 { t.Fatalf("Expected file name length 255, got %v", len(filepath.Base(localFilePath))) } err = clnt.FGetObject(context.Background(), "bucketName", "objectName", localFilePath, GetObjectOptions{}) if err != nil { t.Fatal(err) } buf, err := os.ReadFile(localFilePath) if err != nil { t.Fatalf("Expected 'nil', got %v", err) } if len(buf) != 5 { t.Fatalf("Expected read bytes '5', got %v", len(buf)) } } minio-go-7.0.97/api-get-object.go000066400000000000000000000505411510244170000164540ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "errors" "fmt" "io" "net/http" "sync" "github.com/minio/minio-go/v7/pkg/s3utils" ) // GetObject wrapper function that accepts a request context func (c *Client) GetObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, ErrorResponse{ StatusCode: http.StatusBadRequest, Code: InvalidBucketName, Message: err.Error(), } } if err := s3utils.CheckValidObjectName(objectName); err != nil { return nil, ErrorResponse{ StatusCode: http.StatusBadRequest, Code: XMinioInvalidObjectName, Message: err.Error(), } } gctx, cancel := context.WithCancel(ctx) // Detect if snowball is server location we are talking to. var snowball bool if location, ok := c.bucketLocCache.Get(bucketName); ok { snowball = location == "snowball" } var ( err error httpReader io.ReadCloser objectInfo ObjectInfo totalRead int ) // Create request channel. reqCh := make(chan getRequest) // Create response channel. resCh := make(chan getResponse) // This routine feeds partial object data as and when the caller reads. go func() { defer close(resCh) defer func() { // Close the http response body before returning. // This ends the connection with the server. if httpReader != nil { httpReader.Close() } }() defer cancel() // Used to verify if etag of object has changed since last read. var etag string for req := range reqCh { // If this is the first request we may not need to do a getObject request yet. if req.isFirstReq { // First request is a Read/ReadAt. if req.isReadOp { // Differentiate between wanting the whole object and just a range. if req.isReadAt { // If this is a ReadAt request only get the specified range. // Range is set with respect to the offset and length of the buffer requested. // Do not set objectInfo from the first readAt request because it will not get // the whole object. opts.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1) } else if req.Offset > 0 { opts.SetRange(req.Offset, 0) } httpReader, objectInfo, _, err = c.getObject(gctx, bucketName, objectName, opts) if err != nil { resCh <- getResponse{Error: err} return } etag = objectInfo.ETag // Read at least firstReq.Buffer bytes, if not we have // reached our EOF. size, err := readFull(httpReader, req.Buffer) totalRead += size if size > 0 && err == io.ErrUnexpectedEOF { if int64(size) < objectInfo.Size { // In situations when returned size // is less than the expected content // length set by the server, make sure // we return io.ErrUnexpectedEOF err = io.ErrUnexpectedEOF } else { // If an EOF happens after reading some but not // all the bytes ReadFull returns ErrUnexpectedEOF err = io.EOF } } else if size == 0 && err == io.EOF && objectInfo.Size > 0 { // Special cases when server writes more data // than the content-length, net/http response // body returns an error, instead of converting // it to io.EOF - return unexpected EOF. err = io.ErrUnexpectedEOF } // Send back the first response. resCh <- getResponse{ objectInfo: objectInfo, Size: size, Error: err, didRead: true, } } else { // First request is a Stat or Seek call. // Only need to run a StatObject until an actual Read or ReadAt request comes through. // Remove range header if already set, for stat Operations to get original file size. delete(opts.headers, "Range") objectInfo, err = c.StatObject(gctx, bucketName, objectName, StatObjectOptions(opts)) if err != nil { resCh <- getResponse{ Error: err, } // Exit the go-routine. return } etag = objectInfo.ETag // Send back the first response. resCh <- getResponse{ objectInfo: objectInfo, } } } else if req.settingObjectInfo { // Request is just to get objectInfo. // Remove range header if already set, for stat Operations to get original file size. delete(opts.headers, "Range") // Check whether this is snowball // if yes do not use If-Match feature // it doesn't work. if etag != "" && !snowball { opts.SetMatchETag(etag) } objectInfo, err := c.StatObject(gctx, bucketName, objectName, StatObjectOptions(opts)) if err != nil { resCh <- getResponse{ Error: err, } // Exit the goroutine. return } // Send back the objectInfo. resCh <- getResponse{ objectInfo: objectInfo, } } else { // Offset changes fetch the new object at an Offset. // Because the httpReader may not be set by the first // request if it was a stat or seek it must be checked // if the object has been read or not to only initialize // new ones when they haven't been already. // All readAt requests are new requests. if req.DidOffsetChange || !req.beenRead { // Check whether this is snowball // if yes do not use If-Match feature // it doesn't work. if etag != "" && !snowball { opts.SetMatchETag(etag) } if httpReader != nil { // Close previously opened http reader. httpReader.Close() } // If this request is a readAt only get the specified range. if req.isReadAt { // Range is set with respect to the offset and length of the buffer requested. opts.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1) } else if req.Offset > 0 { // Range is set with respect to the offset. opts.SetRange(req.Offset, 0) } else { // Remove range header if already set delete(opts.headers, "Range") } httpReader, objectInfo, _, err = c.getObject(gctx, bucketName, objectName, opts) if err != nil { resCh <- getResponse{ Error: err, } return } totalRead = 0 } // Read at least req.Buffer bytes, if not we have // reached our EOF. size, err := readFull(httpReader, req.Buffer) totalRead += size if size > 0 && err == io.ErrUnexpectedEOF { if int64(totalRead) < objectInfo.Size { // In situations when returned size // is less than the expected content // length set by the server, make sure // we return io.ErrUnexpectedEOF err = io.ErrUnexpectedEOF } else { // If an EOF happens after reading some but not // all the bytes ReadFull returns ErrUnexpectedEOF err = io.EOF } } else if size == 0 && err == io.EOF && objectInfo.Size > 0 { // Special cases when server writes more data // than the content-length, net/http response // body returns an error, instead of converting // it to io.EOF - return unexpected EOF. err = io.ErrUnexpectedEOF } // Reply back how much was read. resCh <- getResponse{ Size: size, Error: err, didRead: true, objectInfo: objectInfo, } } } }() // Create a newObject through the information sent back by reqCh. return newObject(gctx, cancel, reqCh, resCh), nil } // get request message container to communicate with internal // go-routine. type getRequest struct { Buffer []byte Offset int64 // readAt offset. DidOffsetChange bool // Tracks the offset changes for Seek requests. beenRead bool // Determines if this is the first time an object is being read. isReadAt bool // Determines if this request is a request to a specific range isReadOp bool // Determines if this request is a Read or Read/At request. isFirstReq bool // Determines if this request is the first time an object is being accessed. settingObjectInfo bool // Determines if this request is to set the objectInfo of an object. } // get response message container to reply back for the request. type getResponse struct { Size int Error error didRead bool // Lets subsequent calls know whether or not httpReader has been initiated. objectInfo ObjectInfo // Used for the first request. } // Object represents an open object. It implements // Reader, ReaderAt, Seeker, Closer for a HTTP stream. type Object struct { // Mutex. mutex *sync.Mutex // User allocated and defined. reqCh chan<- getRequest resCh <-chan getResponse ctx context.Context cancel context.CancelFunc currOffset int64 objectInfo ObjectInfo // Ask lower level to initiate data fetching based on currOffset seekData bool // Keeps track of closed call. isClosed bool // Keeps track of if this is the first call. isStarted bool // Previous error saved for future calls. prevErr error // Keeps track of if this object has been read yet. beenRead bool // Keeps track of if objectInfo has been set yet. objectInfoSet bool } // doGetRequest - sends and blocks on the firstReqCh and reqCh of an object. // Returns back the size of the buffer read, if anything was read, as well // as any error encountered. For all first requests sent on the object // it is also responsible for sending back the objectInfo. func (o *Object) doGetRequest(request getRequest) (getResponse, error) { select { case <-o.ctx.Done(): return getResponse{}, o.ctx.Err() case o.reqCh <- request: } response := <-o.resCh // Return any error to the top level. if response.Error != nil && response.Error != io.EOF { return response, response.Error } // This was the first request. if !o.isStarted { // The object has been operated on. o.isStarted = true } // Set the objectInfo if the request was not readAt // and it hasn't been set before. if !o.objectInfoSet && !request.isReadAt { o.objectInfo = response.objectInfo o.objectInfoSet = true } // Set beenRead only if it has not been set before. if !o.beenRead { o.beenRead = response.didRead } // Data are ready on the wire, no need to reinitiate connection in lower level o.seekData = false return response, response.Error } // setOffset - handles the setting of offsets for // Read/ReadAt/Seek requests. func (o *Object) setOffset(bytesRead int64) error { // Update the currentOffset. o.currOffset += bytesRead if o.objectInfo.Size > -1 && o.currOffset >= o.objectInfo.Size { return io.EOF } return nil } // Read reads up to len(b) bytes into b. It returns the number of // bytes read (0 <= n <= len(b)) and any error encountered. Returns // io.EOF upon end of file. func (o *Object) Read(b []byte) (n int, err error) { if o == nil { return 0, errInvalidArgument("Object is nil") } // Locking. o.mutex.Lock() defer o.mutex.Unlock() // prevErr is previous error saved from previous operation. if o.prevErr != nil || o.isClosed { return 0, o.prevErr } // Create a new request. readReq := getRequest{ isReadOp: true, beenRead: o.beenRead, Buffer: b, } // Alert that this is the first request. if !o.isStarted { readReq.isFirstReq = true } // Ask to establish a new data fetch routine based on seekData flag readReq.DidOffsetChange = o.seekData readReq.Offset = o.currOffset // Send and receive from the first request. response, err := o.doGetRequest(readReq) if err != nil && err != io.EOF { // Save the error for future calls. o.prevErr = err return response.Size, err } // Bytes read. bytesRead := int64(response.Size) // Set the new offset. oerr := o.setOffset(bytesRead) if oerr != nil { // Save the error for future calls. o.prevErr = oerr return response.Size, oerr } // Return the response. return response.Size, err } // Stat returns the ObjectInfo structure describing Object. func (o *Object) Stat() (ObjectInfo, error) { if o == nil { return ObjectInfo{}, errInvalidArgument("Object is nil") } // Locking. o.mutex.Lock() defer o.mutex.Unlock() if o.prevErr != nil && o.prevErr != io.EOF || o.isClosed { return ObjectInfo{}, o.prevErr } // This is the first request. if !o.isStarted || !o.objectInfoSet { // Send the request and get the response. _, err := o.doGetRequest(getRequest{ isFirstReq: !o.isStarted, settingObjectInfo: !o.objectInfoSet, }) if err != nil { o.prevErr = err return ObjectInfo{}, err } } return o.objectInfo, nil } // ReadAt reads len(b) bytes from the File starting at byte offset // off. It returns the number of bytes read and the error, if any. // ReadAt always returns a non-nil error when n < len(b). At end of // file, that error is io.EOF. func (o *Object) ReadAt(b []byte, offset int64) (n int, err error) { if o == nil { return 0, errInvalidArgument("Object is nil") } // Locking. o.mutex.Lock() defer o.mutex.Unlock() // prevErr is error which was saved in previous operation. if o.prevErr != nil && o.prevErr != io.EOF || o.isClosed { return 0, o.prevErr } // Set the current offset to ReadAt offset, because the current offset will be shifted at the end of this method. o.currOffset = offset // Can only compare offsets to size when size has been set. if o.objectInfoSet { // If offset is negative than we return io.EOF. // If offset is greater than or equal to object size we return io.EOF. if (o.objectInfo.Size > -1 && offset >= o.objectInfo.Size) || offset < 0 { return 0, io.EOF } } // Create the new readAt request. readAtReq := getRequest{ isReadOp: true, isReadAt: true, DidOffsetChange: true, // Offset always changes. beenRead: o.beenRead, // Set if this is the first request to try and read. Offset: offset, // Set the offset. Buffer: b, } // Alert that this is the first request. if !o.isStarted { readAtReq.isFirstReq = true } // Send and receive from the first request. response, err := o.doGetRequest(readAtReq) if err != nil && err != io.EOF { // Save the error. o.prevErr = err return response.Size, err } // Bytes read. bytesRead := int64(response.Size) // There is no valid objectInfo yet // to compare against for EOF. if !o.objectInfoSet { // Update the currentOffset. o.currOffset += bytesRead } else { // If this was not the first request update // the offsets and compare against objectInfo // for EOF. oerr := o.setOffset(bytesRead) if oerr != nil { o.prevErr = oerr return response.Size, oerr } } return response.Size, err } // Seek sets the offset for the next Read or Write to offset, // interpreted according to whence: 0 means relative to the // origin of the file, 1 means relative to the current offset, // and 2 means relative to the end. // Seek returns the new offset and an error, if any. // // Seeking to a negative offset is an error. Seeking to any positive // offset is legal, subsequent io operations succeed until the // underlying object is not closed. func (o *Object) Seek(offset int64, whence int) (n int64, err error) { if o == nil { return 0, errInvalidArgument("Object is nil") } // Locking. o.mutex.Lock() defer o.mutex.Unlock() // At EOF seeking is legal allow only io.EOF, for any other errors we return. if o.prevErr != nil && o.prevErr != io.EOF { return 0, o.prevErr } // Negative offset is valid for whence of '2'. if offset < 0 && whence != 2 { return 0, errInvalidArgument(fmt.Sprintf("Negative position not allowed for %d", whence)) } // This is the first request. So before anything else // get the ObjectInfo. if !o.isStarted || !o.objectInfoSet { // Create the new Seek request. seekReq := getRequest{ isReadOp: false, Offset: offset, isFirstReq: true, } // Send and receive from the seek request. _, err := o.doGetRequest(seekReq) if err != nil { // Save the error. o.prevErr = err return 0, err } } newOffset := o.currOffset // Switch through whence. switch whence { default: return 0, errInvalidArgument(fmt.Sprintf("Invalid whence %d", whence)) case 0: if o.objectInfo.Size > -1 && offset > o.objectInfo.Size { return 0, io.EOF } newOffset = offset case 1: if o.objectInfo.Size > -1 && o.currOffset+offset > o.objectInfo.Size { return 0, io.EOF } newOffset += offset case 2: // If we don't know the object size return an error for io.SeekEnd if o.objectInfo.Size < 0 { return 0, errInvalidArgument("Whence END is not supported when the object size is unknown") } // Seeking to positive offset is valid for whence '2', but // since we are backing a Reader we have reached 'EOF' if // offset is positive. if offset > 0 { return 0, io.EOF } // Seeking to negative position not allowed for whence. if o.objectInfo.Size+offset < 0 { return 0, errInvalidArgument(fmt.Sprintf("Seeking at negative offset not allowed for %d", whence)) } newOffset = o.objectInfo.Size + offset } // Reset the saved error since we successfully seeked, let the Read // and ReadAt decide. if o.prevErr == io.EOF { o.prevErr = nil } // Ask lower level to fetch again from source when necessary o.seekData = (newOffset != o.currOffset) || o.seekData o.currOffset = newOffset // Return the effective offset. return o.currOffset, nil } // Close - The behavior of Close after the first call returns error // for subsequent Close() calls. func (o *Object) Close() (err error) { if o == nil { return errInvalidArgument("Object is nil") } // Locking. o.mutex.Lock() defer o.mutex.Unlock() // if already closed return an error. if o.isClosed { return o.prevErr } // Close successfully. o.cancel() // Close the request channel to indicate the internal go-routine to exit. close(o.reqCh) // Save for future operations. errMsg := "Object is already closed. Bad file descriptor." o.prevErr = errors.New(errMsg) // Save here that we closed done channel successfully. o.isClosed = true return nil } // newObject instantiates a new *minio.Object* // ObjectInfo will be set by setObjectInfo func newObject(ctx context.Context, cancel context.CancelFunc, reqCh chan<- getRequest, resCh <-chan getResponse) *Object { return &Object{ ctx: ctx, cancel: cancel, mutex: &sync.Mutex{}, reqCh: reqCh, resCh: resCh, } } // getObject - retrieve object from Object Storage. // // Additionally this function also takes range arguments to download the specified // range bytes of an object. Setting offset and length = 0 will download the full object. // // For more information about the HTTP Range header. // go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. func (c *Client) getObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, http.Header, error) { // Validate input arguments. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, ObjectInfo{}, nil, ErrorResponse{ StatusCode: http.StatusBadRequest, Code: InvalidBucketName, Message: err.Error(), } } if err := s3utils.CheckValidObjectName(objectName); err != nil { return nil, ObjectInfo{}, nil, ErrorResponse{ StatusCode: http.StatusBadRequest, Code: XMinioInvalidObjectName, Message: err.Error(), } } // Execute GET on objectName. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: opts.toQueryValues(), customHeader: opts.Header(), contentSHA256Hex: emptySHA256Hex, }) if err != nil { return nil, ObjectInfo{}, nil, err } if resp != nil { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { return nil, ObjectInfo{}, nil, httpRespToErrorResponse(resp, bucketName, objectName) } } objectStat, err := ToObjectInfo(bucketName, objectName, resp.Header) if err != nil { closeResponse(resp) return nil, ObjectInfo{}, nil, err } // do not close body here, caller will close return resp.Body, objectStat, resp.Header, nil } minio-go-7.0.97/api-get-object_test.go000066400000000000000000000101031510244170000175010ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "crypto/rand" "io" "net/http" "net/http/httptest" "testing" ) func TestGetObjectReturnSuccess(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") w.Header().Set("Content-Length", "5") // Write less bytes than the content length. w.Write([]byte("12345")) })) defer srv.Close() // New - instantiate minio client with options clnt, err := New(srv.Listener.Addr().String(), &Options{ Region: "us-east-1", }) if err != nil { t.Fatal(err) } obj, err := clnt.GetObject(context.Background(), "bucketName", "objectName", GetObjectOptions{}) if err != nil { t.Fatal(err) } // We expect an error when reading back. buf, err := io.ReadAll(obj) if err != nil { t.Fatalf("Expected 'nil', got %v", err) } if len(buf) != 5 { t.Fatalf("Expected read bytes '5', got %v", len(buf)) } } func TestGetObjectReturnErrorIfServerTruncatesResponse(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") w.Header().Set("Content-Length", "100") // Write less bytes than the content length. w.Write([]byte("12345")) })) defer srv.Close() // New - instantiate minio client with options clnt, err := New(srv.Listener.Addr().String(), &Options{ Region: "us-east-1", }) if err != nil { t.Fatal(err) } obj, err := clnt.GetObject(context.Background(), "bucketName", "objectName", GetObjectOptions{}) if err != nil { t.Fatal(err) } // We expect an error when reading back. if _, err = io.ReadAll(obj); err != io.ErrUnexpectedEOF { t.Fatalf("Expected %v, got %v", io.ErrUnexpectedEOF, err) } } func TestGetObjectReturnErrorIfServerTruncatesResponseDouble(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") w.Header().Set("Content-Length", "1024") // Write less bytes than the content length. io.Copy(w, io.LimitReader(rand.Reader, 1023)) })) defer srv.Close() // New - instantiate minio client with options clnt, err := New(srv.Listener.Addr().String(), &Options{ Region: "us-east-1", }) if err != nil { t.Fatal(err) } obj, err := clnt.GetObject(context.Background(), "bucketName", "objectName", GetObjectOptions{}) if err != nil { t.Fatal(err) } // We expect an error when reading back. if _, err = io.ReadAll(obj); err != io.ErrUnexpectedEOF { t.Fatalf("Expected %v, got %v", io.ErrUnexpectedEOF, err) } } func TestGetObjectReturnErrorIfServerSendsMore(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") w.Header().Set("Content-Length", "1") // Write less bytes than the content length. w.Write([]byte("12345")) })) defer srv.Close() // New - instantiate minio client with options clnt, err := New(srv.Listener.Addr().String(), &Options{ Region: "us-east-1", }) if err != nil { t.Fatal(err) } obj, err := clnt.GetObject(context.Background(), "bucketName", "objectName", GetObjectOptions{}) if err != nil { t.Fatal(err) } // We expect an error when reading back. if _, err = io.ReadAll(obj); err != io.ErrUnexpectedEOF { t.Fatalf("Expected %v, got %v", io.ErrUnexpectedEOF, err) } } minio-go-7.0.97/api-get-options.go000066400000000000000000000137411510244170000167020ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "fmt" "net/http" "net/url" "strconv" "time" "github.com/minio/minio-go/v7/pkg/encrypt" ) // AdvancedGetOptions for internal use by MinIO server - not intended for client use. type AdvancedGetOptions struct { ReplicationDeleteMarker bool IsReplicationReadyForDeleteMarker bool ReplicationProxyRequest string } // GetObjectOptions are used to specify additional headers or options // during GET requests. type GetObjectOptions struct { headers map[string]string reqParams url.Values ServerSideEncryption encrypt.ServerSide VersionID string PartNumber int // Include any checksums, if object was uploaded with checksum. // For multipart objects this is a checksum of part checksums. // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html Checksum bool // To be not used by external applications Internal AdvancedGetOptions } // StatObjectOptions are used to specify additional headers or options // during GET info/stat requests. type StatObjectOptions = GetObjectOptions // Header returns the http.Header representation of the GET options. func (o GetObjectOptions) Header() http.Header { headers := make(http.Header, len(o.headers)) for k, v := range o.headers { headers.Set(k, v) } if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() == encrypt.SSEC { o.ServerSideEncryption.Marshal(headers) } // this header is set for active-active replication scenario where GET/HEAD // to site A is proxy'd to site B if object/version missing on site A. if o.Internal.ReplicationProxyRequest != "" { headers.Set(minIOBucketReplicationProxyRequest, o.Internal.ReplicationProxyRequest) } if o.Checksum { headers.Set("x-amz-checksum-mode", "ENABLED") } return headers } // Set adds a key value pair to the options. The // key-value pair will be part of the HTTP GET request // headers. func (o *GetObjectOptions) Set(key, value string) { if o.headers == nil { o.headers = make(map[string]string) } o.headers[http.CanonicalHeaderKey(key)] = value } // SetReqParam - set request query string parameter // supported key: see supportedQueryValues and allowedCustomQueryPrefix. // If an unsupported key is passed in, it will be ignored and nothing will be done. func (o *GetObjectOptions) SetReqParam(key, value string) { if !isCustomQueryValue(key) && !isStandardQueryValue(key) { // do nothing return } if o.reqParams == nil { o.reqParams = make(url.Values) } o.reqParams.Set(key, value) } // AddReqParam - add request query string parameter // supported key: see supportedQueryValues and allowedCustomQueryPrefix. // If an unsupported key is passed in, it will be ignored and nothing will be done. func (o *GetObjectOptions) AddReqParam(key, value string) { if !isCustomQueryValue(key) && !isStandardQueryValue(key) { // do nothing return } if o.reqParams == nil { o.reqParams = make(url.Values) } o.reqParams.Add(key, value) } // SetMatchETag - set match etag. func (o *GetObjectOptions) SetMatchETag(etag string) error { if etag == "" { return errInvalidArgument("ETag cannot be empty.") } o.Set("If-Match", "\""+etag+"\"") return nil } // SetMatchETagExcept - set match etag except. func (o *GetObjectOptions) SetMatchETagExcept(etag string) error { if etag == "" { return errInvalidArgument("ETag cannot be empty.") } o.Set("If-None-Match", "\""+etag+"\"") return nil } // SetUnmodified - set unmodified time since. func (o *GetObjectOptions) SetUnmodified(modTime time.Time) error { if modTime.IsZero() { return errInvalidArgument("Modified since cannot be empty.") } o.Set("If-Unmodified-Since", modTime.Format(http.TimeFormat)) return nil } // SetModified - set modified time since. func (o *GetObjectOptions) SetModified(modTime time.Time) error { if modTime.IsZero() { return errInvalidArgument("Modified since cannot be empty.") } o.Set("If-Modified-Since", modTime.Format(http.TimeFormat)) return nil } // SetRange - set the start and end offset of the object to be read. // See https://tools.ietf.org/html/rfc7233#section-3.1 for reference. func (o *GetObjectOptions) SetRange(start, end int64) error { switch { case start == 0 && end < 0: // Read last '-end' bytes. `bytes=-N`. o.Set("Range", fmt.Sprintf("bytes=%d", end)) case 0 < start && end == 0: // Read everything starting from offset // 'start'. `bytes=N-`. o.Set("Range", fmt.Sprintf("bytes=%d-", start)) case 0 <= start && start <= end: // Read everything starting at 'start' till the // 'end'. `bytes=N-M` o.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) default: // All other cases such as // bytes=-3- // bytes=5-3 // bytes=-2-4 // bytes=-3-0 // bytes=-3--2 // are invalid. return errInvalidArgument( fmt.Sprintf( "Invalid range specified: start=%d end=%d", start, end)) } return nil } // toQueryValues - Convert the versionId, partNumber, and reqParams in Options to query string parameters. func (o *GetObjectOptions) toQueryValues() url.Values { urlValues := make(url.Values) if o.VersionID != "" { urlValues.Set("versionId", o.VersionID) } if o.PartNumber > 0 { urlValues.Set("partNumber", strconv.Itoa(o.PartNumber)) } if o.reqParams != nil { for key, values := range o.reqParams { for _, value := range values { urlValues.Add(key, value) } } } return urlValues } minio-go-7.0.97/api-inventory-ext.go000066400000000000000000000267651510244170000172770ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2025 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "encoding/json" "io" "iter" "net/http" "net/url" "strings" "time" "github.com/minio/minio-go/v7/pkg/s3utils" ) // This file contains the inventory API extension for MinIO server. It is not // compatible with AWS S3. func makeInventoryReqMetadata(bucket string, urlParams ...string) requestMetadata { urlValues := make(url.Values) urlValues.Set("minio-inventory", "") // If an odd number of parameters is given, we skip the last pair to avoid // an out of bounds access. for i := 0; i+1 < len(urlParams); i += 2 { urlValues.Set(urlParams[i], urlParams[i+1]) } return requestMetadata{ bucketName: bucket, queryValues: urlValues, } } // GenerateInventoryConfigYAML generates a YAML template for an inventory configuration. // This is a MinIO-specific API and is not compatible with AWS S3. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucket: Name of the bucket // - id: Unique identifier for the inventory configuration // // Returns a YAML template string that can be customized and used with PutBucketInventoryConfiguration. func (c *Client) GenerateInventoryConfigYAML(ctx context.Context, bucket, id string) (string, error) { if err := s3utils.CheckValidBucketName(bucket); err != nil { return "", err } if id == "" { return "", errInvalidArgument("inventory ID cannot be empty") } reqMeta := makeInventoryReqMetadata(bucket, "generate", "", "id", id) resp, err := c.executeMethod(ctx, http.MethodGet, reqMeta) defer closeResponse(resp) if err != nil { return "", err } if resp.StatusCode != http.StatusOK { return "", httpRespToErrorResponse(resp, bucket, "") } buf := new(strings.Builder) _, err = io.Copy(buf, resp.Body) return buf.String(), err } // inventoryPutConfigOpts is a placeholder for future options that may be added. type inventoryPutConfigOpts struct{} // InventoryPutConfigOption is to allow for functional options for // PutBucketInventoryConfiguration. It may be used in the future to customize // the PutBucketInventoryConfiguration request, but currently does not do // anything. type InventoryPutConfigOption func(*inventoryPutConfigOpts) // PutBucketInventoryConfiguration creates or updates an inventory configuration for a bucket. // This is a MinIO-specific API and is not compatible with AWS S3. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucket: Name of the bucket // - id: Unique identifier for the inventory configuration // - yamlDef: YAML definition of the inventory configuration // // Returns an error if the operation fails, or if bucket name, id, or yamlDef is empty. func (c *Client) PutBucketInventoryConfiguration(ctx context.Context, bucket string, id string, yamlDef string, _ ...InventoryPutConfigOption) error { if err := s3utils.CheckValidBucketName(bucket); err != nil { return err } if id == "" { return errInvalidArgument("inventory ID cannot be empty") } if yamlDef == "" { return errInvalidArgument("YAML definition cannot be empty") } reqMeta := makeInventoryReqMetadata(bucket, "id", id) reqMeta.contentBody = strings.NewReader(yamlDef) reqMeta.contentLength = int64(len(yamlDef)) reqMeta.contentMD5Base64 = sumMD5Base64([]byte(yamlDef)) resp, err := c.executeMethod(ctx, http.MethodPut, reqMeta) defer closeResponse(resp) if err != nil { return err } if resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucket, "") } return nil } // GetBucketInventoryConfiguration retrieves the inventory configuration for a bucket. // This is a MinIO-specific API and is not compatible with AWS S3. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucket: Name of the bucket // - id: Unique identifier for the inventory configuration // // Returns the inventory configuration or an error if the operation fails or if the configuration doesn't exist. func (c *Client) GetBucketInventoryConfiguration(ctx context.Context, bucket, id string) (*InventoryConfiguration, error) { if err := s3utils.CheckValidBucketName(bucket); err != nil { return nil, err } if id == "" { return nil, errInvalidArgument("inventory ID cannot be empty") } reqMeta := makeInventoryReqMetadata(bucket, "id", id) resp, err := c.executeMethod(ctx, http.MethodGet, reqMeta) defer closeResponse(resp) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, bucket, "") } decoder := json.NewDecoder(resp.Body) var ic InventoryConfiguration err = decoder.Decode(&ic) if err != nil { return nil, err } return &ic, nil } // DeleteBucketInventoryConfiguration deletes an inventory configuration from a bucket. // This is a MinIO-specific API and is not compatible with AWS S3. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucket: Name of the bucket // - id: Unique identifier for the inventory configuration to delete // // Returns an error if the operation fails or if the configuration doesn't exist. func (c *Client) DeleteBucketInventoryConfiguration(ctx context.Context, bucket, id string) error { if err := s3utils.CheckValidBucketName(bucket); err != nil { return err } if id == "" { return errInvalidArgument("inventory ID cannot be empty") } reqMeta := makeInventoryReqMetadata(bucket, "id", id) resp, err := c.executeMethod(ctx, http.MethodDelete, reqMeta) defer closeResponse(resp) if err != nil { return err } if resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucket, "") } return nil } // InventoryConfiguration represents the inventory configuration type InventoryConfiguration struct { Bucket string `json:"bucket"` ID string `json:"id"` User string `json:"user"` YamlDef string `json:"yamlDef,omitempty"` } // InventoryListResult represents the result of listing inventory // configurations. type InventoryListResult struct { Items []InventoryConfiguration `json:"items"` NextContinuationToken string `json:"nextContinuationToken,omitempty"` } // ListBucketInventoryConfigurations lists up to 100 inventory configurations for a bucket. // This is a MinIO-specific API and is not compatible with AWS S3. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucket: Name of the bucket // - continuationToken: Token for pagination (empty string for first request) // // Returns a list result with configurations and a continuation token for the next page, or an error. func (c *Client) ListBucketInventoryConfigurations(ctx context.Context, bucket, continuationToken string) (lr *InventoryListResult, err error) { if err := s3utils.CheckValidBucketName(bucket); err != nil { return nil, err } reqMeta := makeInventoryReqMetadata(bucket, "continuation-token", continuationToken) resp, err := c.executeMethod(ctx, http.MethodGet, reqMeta) defer closeResponse(resp) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, bucket, "") } decoder := json.NewDecoder(resp.Body) err = decoder.Decode(&lr) if err != nil { return nil, err } return lr, nil } // ListBucketInventoryConfigurationsIterator returns an iterator that lists all inventory configurations // for a bucket. This is a MinIO-specific API and is not compatible with AWS S3. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucket: Name of the bucket // // Returns an iterator that yields InventoryConfiguration values and errors. The iterator automatically // handles pagination and fetches all configurations. func (c *Client) ListBucketInventoryConfigurationsIterator(ctx context.Context, bucket string) iter.Seq2[InventoryConfiguration, error] { return func(yield func(InventoryConfiguration, error) bool) { if err := s3utils.CheckValidBucketName(bucket); err != nil { yield(InventoryConfiguration{}, err) return } var continuationToken string for { listResult, err := c.ListBucketInventoryConfigurations(ctx, bucket, continuationToken) if err != nil { yield(InventoryConfiguration{}, err) return } for _, item := range listResult.Items { if !yield(item, nil) { return } } if listResult.NextContinuationToken == "" { return } continuationToken = listResult.NextContinuationToken } } } // InventoryJobStatus represents the status of an inventory job. type InventoryJobStatus struct { Bucket string `json:"bucket"` ID string `json:"id"` User string `json:"user"` AccessKey string `json:"accessKey"` Schedule string `json:"schedule"` State string `json:"state"` NextScheduledTime time.Time `json:"nextScheduledTime,omitempty"` StartTime time.Time `json:"startTime,omitempty"` EndTime time.Time `json:"endTime,omitempty"` LastUpdate time.Time `json:"lastUpdate,omitempty"` Scanned string `json:"scanned,omitempty"` Matched string `json:"matched,omitempty"` ScannedCount uint64 `json:"scannedCount,omitempty"` MatchedCount uint64 `json:"matchedCount,omitempty"` RecordsWritten uint64 `json:"recordsWritten,omitempty"` OutputFilesCount uint64 `json:"outputFilesCount,omitempty"` ExecutionTime string `json:"executionTime,omitempty"` NumStarts uint64 `json:"numStarts,omitempty"` NumErrors uint64 `json:"numErrors,omitempty"` NumLockLosses uint64 `json:"numLockLosses,omitempty"` ManifestPath string `json:"manifestPath,omitempty"` RetryAttempts uint64 `json:"retryAttempts,omitempty"` LastFailTime time.Time `json:"lastFailTime,omitempty"` LastFailErrors []string `json:"lastFailErrors,omitempty"` } // GetBucketInventoryJobStatus retrieves the status of an inventory job for a bucket. // This is a MinIO-specific API and is not compatible with AWS S3. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucket: Name of the bucket // - id: Unique identifier for the inventory job // // Returns the inventory job status including execution state, progress, and error information, or an error if the operation fails. func (c *Client) GetBucketInventoryJobStatus(ctx context.Context, bucket, id string) (*InventoryJobStatus, error) { if err := s3utils.CheckValidBucketName(bucket); err != nil { return nil, err } if id == "" { return nil, errInvalidArgument("inventory ID cannot be empty") } reqMeta := makeInventoryReqMetadata(bucket, "id", id, "status", "") resp, err := c.executeMethod(ctx, http.MethodGet, reqMeta) defer closeResponse(resp) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, bucket, "") } decoder := json.NewDecoder(resp.Body) var jStatus InventoryJobStatus err = decoder.Decode(&jStatus) if err != nil { return nil, err } return &jStatus, nil } minio-go-7.0.97/api-list.go000066400000000000000000001047721510244170000154120ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "fmt" "iter" "net/http" "net/url" "slices" "time" "github.com/minio/minio-go/v7/pkg/s3utils" ) // ListBuckets list all buckets owned by this authenticated user. // // This call requires explicit authentication, no anonymous requests are // allowed for listing buckets. // // api := client.New(....) // for message := range api.ListBuckets(context.Background()) { // fmt.Println(message) // } func (c *Client) ListBuckets(ctx context.Context) ([]BucketInfo, error) { // Execute GET on service. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{contentSHA256Hex: emptySHA256Hex}) defer closeResponse(resp) if err != nil { return nil, err } if resp != nil { if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, "", "") } } listAllMyBucketsResult := listAllMyBucketsResult{} err = xmlDecoder(resp.Body, &listAllMyBucketsResult) if err != nil { return nil, err } return listAllMyBucketsResult.Buckets.Bucket, nil } // ListDirectoryBuckets list all buckets owned by this authenticated user. // // This call requires explicit authentication, no anonymous requests are // allowed for listing buckets. // // api := client.New(....) // dirBuckets, err := api.ListDirectoryBuckets(context.Background()) func (c *Client) ListDirectoryBuckets(ctx context.Context) (iter.Seq2[BucketInfo, error], error) { fetchBuckets := func(continuationToken string) ([]BucketInfo, string, error) { metadata := requestMetadata{contentSHA256Hex: emptySHA256Hex} metadata.queryValues = url.Values{} metadata.queryValues.Set("max-directory-buckets", "1000") if continuationToken != "" { metadata.queryValues.Set("continuation-token", continuationToken) } // Execute GET on service. resp, err := c.executeMethod(ctx, http.MethodGet, metadata) defer closeResponse(resp) if err != nil { return nil, "", err } if resp != nil { if resp.StatusCode != http.StatusOK { return nil, "", httpRespToErrorResponse(resp, "", "") } } results := listAllMyDirectoryBucketsResult{} if err = xmlDecoder(resp.Body, &results); err != nil { return nil, "", err } return results.Buckets.Bucket, results.ContinuationToken, nil } return func(yield func(BucketInfo, error) bool) { var continuationToken string for { buckets, token, err := fetchBuckets(continuationToken) if err != nil { yield(BucketInfo{}, err) return } for _, bucket := range buckets { if !yield(bucket, nil) { return } } if token == "" { // nothing to continue return } continuationToken = token } }, nil } // Bucket List Operations. func (c *Client) listObjectsV2(ctx context.Context, bucketName string, opts ListObjectsOptions) iter.Seq[ObjectInfo] { // Default listing is delimited at "/" delimiter := "/" if opts.Recursive { // If recursive we do not delimit. delimiter = "" } // Return object owner information by default fetchOwner := true return func(yield func(ObjectInfo) bool) { if contextCanceled(ctx) { return } // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { yield(ObjectInfo{Err: err}) return } // Validate incoming object prefix. if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil { yield(ObjectInfo{Err: err}) return } // Save continuationToken for next request. var continuationToken string for { if contextCanceled(ctx) { return } // Get list of objects a maximum of 1000 per request. result, err := c.listObjectsV2Query(ctx, bucketName, opts.Prefix, continuationToken, fetchOwner, opts.WithMetadata, delimiter, opts.StartAfter, opts.MaxKeys, opts.headers) if err != nil { yield(ObjectInfo{Err: err}) return } // If contents are available loop through and send over channel. for _, object := range result.Contents { object.ETag = trimEtag(object.ETag) if !yield(object) { return } } // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { if !yield(ObjectInfo{Key: obj.Prefix}) { return } } // If continuation token present, save it for next request. if result.NextContinuationToken != "" { continuationToken = result.NextContinuationToken } // Listing ends result is not truncated, return right here. if !result.IsTruncated { return } // Add this to catch broken S3 API implementations. if continuationToken == "" { if !yield(ObjectInfo{ Err: fmt.Errorf("listObjectsV2 is truncated without continuationToken, %s S3 server is buggy", c.endpointURL), }) { return } } } } } // listObjectsV2Query - (List Objects V2) - List some or all (up to 1000) of the objects in a bucket. // // You can use the request parameters as selection criteria to return a subset of the objects in a bucket. // request parameters :- // --------- // ?prefix - Limits the response to keys that begin with the specified prefix. // ?continuation-token - Used to continue iterating over a set of objects // ?metadata - Specifies if we want metadata for the objects as part of list operation. // ?delimiter - A delimiter is a character you use to group keys. // ?start-after - Sets a marker to start listing lexically at this key onwards. // ?max-keys - Sets the maximum number of keys returned in the response body. func (c *Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter, startAfter string, maxkeys int, headers http.Header) (ListBucketV2Result, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListBucketV2Result{}, err } // Validate object prefix. if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { return ListBucketV2Result{}, err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) // Always set list-type in ListObjects V2 urlValues.Set("list-type", "2") if metadata { urlValues.Set("metadata", "true") } // Set this conditionally if asked if startAfter != "" { urlValues.Set("start-after", startAfter) } // Always set encoding-type in ListObjects V2 urlValues.Set("encoding-type", "url") // Set object prefix, prefix value to be set to empty is okay. urlValues.Set("prefix", objectPrefix) // Set delimiter, delimiter value to be set to empty is okay. urlValues.Set("delimiter", delimiter) // Set continuation token if continuationToken != "" { urlValues.Set("continuation-token", continuationToken) } // Fetch owner when listing if fetchOwner { urlValues.Set("fetch-owner", "true") } // Set max keys. if maxkeys > 0 { urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) } // Execute GET on bucket to list objects. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, customHeader: headers, }) defer closeResponse(resp) if err != nil { return ListBucketV2Result{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { return ListBucketV2Result{}, httpRespToErrorResponse(resp, bucketName, "") } } // Decode listBuckets XML. listBucketResult := ListBucketV2Result{} if err = xmlDecoder(resp.Body, &listBucketResult); err != nil { return listBucketResult, err } // This is an additional verification check to make // sure proper responses are received. if listBucketResult.IsTruncated && listBucketResult.NextContinuationToken == "" { return listBucketResult, ErrorResponse{ Code: NotImplemented, Message: "Truncated response should have continuation token set", } } for i, obj := range listBucketResult.Contents { listBucketResult.Contents[i].Key, err = decodeS3Name(obj.Key, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } listBucketResult.Contents[i].LastModified = listBucketResult.Contents[i].LastModified.Truncate(time.Millisecond) } for i, obj := range listBucketResult.CommonPrefixes { listBucketResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } } // Success. return listBucketResult, nil } func (c *Client) listObjects(ctx context.Context, bucketName string, opts ListObjectsOptions) iter.Seq[ObjectInfo] { // Default listing is delimited at "/" delimiter := "/" if opts.Recursive { // If recursive we do not delimit. delimiter = "" } return func(yield func(ObjectInfo) bool) { if contextCanceled(ctx) { return } // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { yield(ObjectInfo{Err: err}) return } // Validate incoming object prefix. if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil { yield(ObjectInfo{Err: err}) return } marker := opts.StartAfter for { if contextCanceled(ctx) { return } // Get list of objects a maximum of 1000 per request. result, err := c.listObjectsQuery(ctx, bucketName, opts.Prefix, marker, delimiter, opts.MaxKeys, opts.headers) if err != nil { yield(ObjectInfo{Err: err}) return } // If contents are available loop through and send over channel. for _, object := range result.Contents { // Save the marker. marker = object.Key object.ETag = trimEtag(object.ETag) if !yield(object) { return } } // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { if !yield(ObjectInfo{Key: obj.Prefix}) { return } } // If next marker present, save it for next request. if result.NextMarker != "" { marker = result.NextMarker } // Listing ends result is not truncated, return right here. if !result.IsTruncated { return } } } } func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts ListObjectsOptions) iter.Seq[ObjectInfo] { // Default listing is delimited at "/" delimiter := "/" if opts.Recursive { // If recursive we do not delimit. delimiter = "" } return func(yield func(ObjectInfo) bool) { if contextCanceled(ctx) { return } // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { yield(ObjectInfo{Err: err}) return } // Validate incoming object prefix. if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil { yield(ObjectInfo{Err: err}) return } var ( keyMarker = "" versionIDMarker = "" preName = "" preKey = "" perVersions []Version numVersions int ) send := func(vers []Version) bool { if opts.WithVersions && opts.ReverseVersions { slices.Reverse(vers) numVersions = len(vers) } for _, version := range vers { info := ObjectInfo{ ETag: trimEtag(version.ETag), Key: version.Key, LastModified: version.LastModified.Truncate(time.Millisecond), Size: version.Size, Owner: version.Owner, StorageClass: version.StorageClass, IsLatest: version.IsLatest, VersionID: version.VersionID, IsDeleteMarker: version.isDeleteMarker, UserTags: version.UserTags, UserMetadata: version.UserMetadata, Internal: version.Internal, NumVersions: numVersions, ChecksumMode: version.ChecksumType, ChecksumCRC32: version.ChecksumCRC32, ChecksumCRC32C: version.ChecksumCRC32C, ChecksumSHA1: version.ChecksumSHA1, ChecksumSHA256: version.ChecksumSHA256, ChecksumCRC64NVME: version.ChecksumCRC64NVME, } if !yield(info) { return false } } return true } for { if contextCanceled(ctx) { return } // Get list of objects a maximum of 1000 per request. result, err := c.listObjectVersionsQuery(ctx, bucketName, opts, keyMarker, versionIDMarker, delimiter) if err != nil { yield(ObjectInfo{Err: err}) return } if opts.WithVersions && opts.ReverseVersions { for _, version := range result.Versions { if preName == "" { preName = result.Name preKey = version.Key } if result.Name == preName && preKey == version.Key { // If the current name is same as previous name, // we need to append the version to the previous version. perVersions = append(perVersions, version) continue } // Send the file versions. if !send(perVersions) { return } perVersions = perVersions[:0] perVersions = append(perVersions, version) preName = result.Name preKey = version.Key } } else { if !send(result.Versions) { return } } // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { if !yield(ObjectInfo{Key: obj.Prefix}) { return } } // If next key marker is present, save it for next request. if result.NextKeyMarker != "" { keyMarker = result.NextKeyMarker } // If next version id marker is present, save it for next request. if result.NextVersionIDMarker != "" { versionIDMarker = result.NextVersionIDMarker } // Listing ends result is not truncated, return right here. if !result.IsTruncated { // sent the lasted file with versions if opts.ReverseVersions && len(perVersions) > 0 { if !send(perVersions) { return } } return } } } } // listObjectVersions - (List Object Versions) - List some or all (up to 1000) of the existing objects // and their versions in a bucket. // // You can use the request parameters as selection criteria to return a subset of the objects in a bucket. // request parameters :- // --------- // ?key-marker - Specifies the key to start with when listing objects in a bucket. // ?version-id-marker - Specifies the version id marker to start with when listing objects with versions in a bucket. // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. func (c *Client) listObjectVersionsQuery(ctx context.Context, bucketName string, opts ListObjectsOptions, keyMarker, versionIDMarker, delimiter string) (ListVersionsResult, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListVersionsResult{}, err } // Validate object prefix. if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil { return ListVersionsResult{}, err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) // Set versions to trigger versioning API urlValues.Set("versions", "") // Set object prefix, prefix value to be set to empty is okay. urlValues.Set("prefix", opts.Prefix) // Set delimiter, delimiter value to be set to empty is okay. urlValues.Set("delimiter", delimiter) // Set object marker. if keyMarker != "" { urlValues.Set("key-marker", keyMarker) } // Set max keys. if opts.MaxKeys > 0 { urlValues.Set("max-keys", fmt.Sprintf("%d", opts.MaxKeys)) } // Set version ID marker if versionIDMarker != "" { urlValues.Set("version-id-marker", versionIDMarker) } if opts.WithMetadata { urlValues.Set("metadata", "true") } // Always set encoding-type urlValues.Set("encoding-type", "url") // Execute GET on bucket to list objects. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, customHeader: opts.headers, }) defer closeResponse(resp) if err != nil { return ListVersionsResult{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { return ListVersionsResult{}, httpRespToErrorResponse(resp, bucketName, "") } } // Decode ListVersionsResult XML. listObjectVersionsOutput := ListVersionsResult{} err = xmlDecoder(resp.Body, &listObjectVersionsOutput) if err != nil { return ListVersionsResult{}, err } for i, obj := range listObjectVersionsOutput.Versions { listObjectVersionsOutput.Versions[i].Key, err = decodeS3Name(obj.Key, listObjectVersionsOutput.EncodingType) if err != nil { return listObjectVersionsOutput, err } } for i, obj := range listObjectVersionsOutput.CommonPrefixes { listObjectVersionsOutput.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listObjectVersionsOutput.EncodingType) if err != nil { return listObjectVersionsOutput, err } } if listObjectVersionsOutput.NextKeyMarker != "" { listObjectVersionsOutput.NextKeyMarker, err = decodeS3Name(listObjectVersionsOutput.NextKeyMarker, listObjectVersionsOutput.EncodingType) if err != nil { return listObjectVersionsOutput, err } } return listObjectVersionsOutput, nil } // listObjects - (List Objects) - List some or all (up to 1000) of the objects in a bucket. // // You can use the request parameters as selection criteria to return a subset of the objects in a bucket. // request parameters :- // --------- // ?marker - Specifies the key to start with when listing objects in a bucket. // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. func (c *Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int, headers http.Header) (ListBucketResult, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListBucketResult{}, err } // Validate object prefix. if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { return ListBucketResult{}, err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) // Set object prefix, prefix value to be set to empty is okay. urlValues.Set("prefix", objectPrefix) // Set delimiter, delimiter value to be set to empty is okay. urlValues.Set("delimiter", delimiter) // Set object marker. if objectMarker != "" { urlValues.Set("marker", objectMarker) } // Set max keys. if maxkeys > 0 { urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys)) } // Always set encoding-type urlValues.Set("encoding-type", "url") // Execute GET on bucket to list objects. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, customHeader: headers, }) defer closeResponse(resp) if err != nil { return ListBucketResult{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { return ListBucketResult{}, httpRespToErrorResponse(resp, bucketName, "") } } // Decode listBuckets XML. listBucketResult := ListBucketResult{} err = xmlDecoder(resp.Body, &listBucketResult) if err != nil { return listBucketResult, err } for i, obj := range listBucketResult.Contents { listBucketResult.Contents[i].Key, err = decodeS3Name(obj.Key, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } listBucketResult.Contents[i].LastModified = listBucketResult.Contents[i].LastModified.Truncate(time.Millisecond) } for i, obj := range listBucketResult.CommonPrefixes { listBucketResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } } if listBucketResult.NextMarker != "" { listBucketResult.NextMarker, err = decodeS3Name(listBucketResult.NextMarker, listBucketResult.EncodingType) if err != nil { return listBucketResult, err } } return listBucketResult, nil } // ListObjectsOptions holds all options of a list object request type ListObjectsOptions struct { // ReverseVersions - reverse the order of the object versions ReverseVersions bool // Include objects versions in the listing WithVersions bool // Include objects metadata in the listing WithMetadata bool // Only list objects with the prefix Prefix string // Ignore '/' delimiter Recursive bool // The maximum number of objects requested per // batch, advanced use-case not useful for most // applications MaxKeys int // StartAfter start listing lexically at this // object onwards, this value can also be set // for Marker when `UseV1` is set to true. StartAfter string // Use the deprecated list objects V1 API UseV1 bool headers http.Header } // Set adds a key value pair to the options. The // key-value pair will be part of the HTTP GET request // headers. func (o *ListObjectsOptions) Set(key, value string) { if o.headers == nil { o.headers = make(http.Header) } o.headers.Set(key, value) } // ListObjects returns objects list after evaluating the passed options. // // api := client.New(....) // for object := range api.ListObjects(ctx, "mytestbucket", minio.ListObjectsOptions{Prefix: "starthere", Recursive:true}) { // fmt.Println(object) // } // // If caller cancels the context, then the last entry on the 'chan ObjectInfo' will be the context.Error() // caller must drain the channel entirely and wait until channel is closed before proceeding, without // waiting on the channel to be closed completely you might leak goroutines. func (c *Client) ListObjects(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo { objectStatCh := make(chan ObjectInfo, 1) go func() { defer close(objectStatCh) if contextCanceled(ctx) { objectStatCh <- ObjectInfo{Err: ctx.Err()} return } var objIter iter.Seq[ObjectInfo] switch { case opts.WithVersions: objIter = c.listObjectVersions(ctx, bucketName, opts) case opts.UseV1: objIter = c.listObjects(ctx, bucketName, opts) default: location, _ := c.bucketLocCache.Get(bucketName) if location == "snowball" { objIter = c.listObjects(ctx, bucketName, opts) } else { objIter = c.listObjectsV2(ctx, bucketName, opts) } } for obj := range objIter { select { case <-ctx.Done(): objectStatCh <- ObjectInfo{Err: ctx.Err()} return case objectStatCh <- obj: } } }() return objectStatCh } // ListObjectsIter returns object list as a iterator sequence. // caller must cancel the context if they are not interested in // iterating further, if no more entries the iterator will // automatically stop. // // api := client.New(....) // for object := range api.ListObjectsIter(ctx, "mytestbucket", minio.ListObjectsOptions{Prefix: "starthere", Recursive:true}) { // if object.Err != nil { // // handle the errors. // } // fmt.Println(object) // } // // Canceling the context the iterator will stop, if you wish to discard the yielding make sure // to cancel the passed context without that you might leak coroutines func (c *Client) ListObjectsIter(ctx context.Context, bucketName string, opts ListObjectsOptions) iter.Seq[ObjectInfo] { if opts.WithVersions { return c.listObjectVersions(ctx, bucketName, opts) } // Use legacy list objects v1 API if opts.UseV1 { return c.listObjects(ctx, bucketName, opts) } // Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1. if location, ok := c.bucketLocCache.Get(bucketName); ok { if location == "snowball" { return c.listObjects(ctx, bucketName, opts) } } return c.listObjectsV2(ctx, bucketName, opts) } // ListIncompleteUploads - List incompletely uploaded multipart objects. // // ListIncompleteUploads lists all incompleted objects matching the // objectPrefix from the specified bucket. If recursion is enabled // it would list all subdirectories and all its contents. // // Your input parameters are just bucketName, objectPrefix, recursive. // If you enable recursive as 'true' this function will return back all // the multipart objects in a given bucket name. // // api := client.New(....) // // Recurively list all objects in 'mytestbucket' // recursive := true // for message := range api.ListIncompleteUploads(context.Background(), "mytestbucket", "starthere", recursive) { // fmt.Println(message) // } func (c *Client) ListIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive bool) <-chan ObjectMultipartInfo { return c.listIncompleteUploads(ctx, bucketName, objectPrefix, recursive) } // contextCanceled returns whether a context is canceled. func contextCanceled(ctx context.Context) bool { select { case <-ctx.Done(): return true default: return false } } // listIncompleteUploads lists all incomplete uploads. func (c *Client) listIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive bool) <-chan ObjectMultipartInfo { // Allocate channel for multipart uploads. objectMultipartStatCh := make(chan ObjectMultipartInfo, 1) // Delimiter is set to "/" by default. delimiter := "/" if recursive { // If recursive do not delimit. delimiter = "" } // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { defer close(objectMultipartStatCh) objectMultipartStatCh <- ObjectMultipartInfo{ Err: err, } return objectMultipartStatCh } // Validate incoming object prefix. if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil { defer close(objectMultipartStatCh) objectMultipartStatCh <- ObjectMultipartInfo{ Err: err, } return objectMultipartStatCh } go func(objectMultipartStatCh chan<- ObjectMultipartInfo) { defer func() { if contextCanceled(ctx) { objectMultipartStatCh <- ObjectMultipartInfo{ Err: ctx.Err(), } } close(objectMultipartStatCh) }() // object and upload ID marker for future requests. var objectMarker string var uploadIDMarker string for { // list all multipart uploads. result, err := c.listMultipartUploadsQuery(ctx, bucketName, objectMarker, uploadIDMarker, objectPrefix, delimiter, 0) if err != nil { objectMultipartStatCh <- ObjectMultipartInfo{ Err: err, } return } objectMarker = result.NextKeyMarker uploadIDMarker = result.NextUploadIDMarker // Send all multipart uploads. for _, obj := range result.Uploads { // Calculate total size of the uploaded parts if 'aggregateSize' is enabled. select { // Send individual uploads here. case objectMultipartStatCh <- obj: // If the context is canceled case <-ctx.Done(): return } } // Send all common prefixes if any. // NOTE: prefixes are only present if the request is delimited. for _, obj := range result.CommonPrefixes { select { // Send delimited prefixes here. case objectMultipartStatCh <- ObjectMultipartInfo{Key: obj.Prefix, Size: 0}: // If context is canceled. case <-ctx.Done(): return } } // Listing ends if result not truncated, return right here. if !result.IsTruncated { return } } }(objectMultipartStatCh) // return. return objectMultipartStatCh } // listMultipartUploadsQuery - (List Multipart Uploads). // - Lists some or all (up to 1000) in-progress multipart uploads in a bucket. // // You can use the request parameters as selection criteria to return a subset of the uploads in a bucket. // request parameters. :- // --------- // ?key-marker - Specifies the multipart upload after which listing should begin. // ?upload-id-marker - Together with key-marker specifies the multipart upload after which listing should begin. // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-uploads - Sets the maximum number of multipart uploads returned in the response body. func (c *Client) listMultipartUploadsQuery(ctx context.Context, bucketName, keyMarker, uploadIDMarker, prefix, delimiter string, maxUploads int) (ListMultipartUploadsResult, error) { // Get resources properly escaped and lined up before using them in http request. urlValues := make(url.Values) // Set uploads. urlValues.Set("uploads", "") // Set object key marker. if keyMarker != "" { urlValues.Set("key-marker", keyMarker) } // Set upload id marker. if uploadIDMarker != "" { urlValues.Set("upload-id-marker", uploadIDMarker) } // Set object prefix, prefix value to be set to empty is okay. urlValues.Set("prefix", prefix) // Set delimiter, delimiter value to be set to empty is okay. urlValues.Set("delimiter", delimiter) // Always set encoding-type urlValues.Set("encoding-type", "url") // maxUploads should be 1000 or less. if maxUploads > 0 { // Set max-uploads. urlValues.Set("max-uploads", fmt.Sprintf("%d", maxUploads)) } // Execute GET on bucketName to list multipart uploads. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return ListMultipartUploadsResult{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { return ListMultipartUploadsResult{}, httpRespToErrorResponse(resp, bucketName, "") } } // Decode response body. listMultipartUploadsResult := ListMultipartUploadsResult{} err = xmlDecoder(resp.Body, &listMultipartUploadsResult) if err != nil { return listMultipartUploadsResult, err } listMultipartUploadsResult.NextKeyMarker, err = decodeS3Name(listMultipartUploadsResult.NextKeyMarker, listMultipartUploadsResult.EncodingType) if err != nil { return listMultipartUploadsResult, err } listMultipartUploadsResult.NextUploadIDMarker, err = decodeS3Name(listMultipartUploadsResult.NextUploadIDMarker, listMultipartUploadsResult.EncodingType) if err != nil { return listMultipartUploadsResult, err } for i, obj := range listMultipartUploadsResult.Uploads { listMultipartUploadsResult.Uploads[i].Key, err = decodeS3Name(obj.Key, listMultipartUploadsResult.EncodingType) if err != nil { return listMultipartUploadsResult, err } } for i, obj := range listMultipartUploadsResult.CommonPrefixes { listMultipartUploadsResult.CommonPrefixes[i].Prefix, err = decodeS3Name(obj.Prefix, listMultipartUploadsResult.EncodingType) if err != nil { return listMultipartUploadsResult, err } } return listMultipartUploadsResult, nil } // listObjectParts list all object parts recursively. // //lint:ignore U1000 Keep this around func (c *Client) listObjectParts(ctx context.Context, bucketName, objectName, uploadID string) (partsInfo map[int]ObjectPart, err error) { // Part number marker for the next batch of request. var nextPartNumberMarker int partsInfo = make(map[int]ObjectPart) for { // Get list of uploaded parts a maximum of 1000 per request. listObjPartsResult, err := c.listObjectPartsQuery(ctx, bucketName, objectName, uploadID, nextPartNumberMarker, 1000) if err != nil { return nil, err } // Append to parts info. for _, part := range listObjPartsResult.ObjectParts { // Trim off the odd double quotes from ETag in the beginning and end. part.ETag = trimEtag(part.ETag) partsInfo[part.PartNumber] = part } // Keep part number marker, for the next iteration. nextPartNumberMarker = listObjPartsResult.NextPartNumberMarker // Listing ends result is not truncated, return right here. if !listObjPartsResult.IsTruncated { break } } // Return all the parts. return partsInfo, nil } // findUploadIDs lists all incomplete uploads and find the uploadIDs of the matching object name. func (c *Client) findUploadIDs(ctx context.Context, bucketName, objectName string) ([]string, error) { var uploadIDs []string // Make list incomplete uploads recursive. isRecursive := true // List all incomplete uploads. for mpUpload := range c.listIncompleteUploads(ctx, bucketName, objectName, isRecursive) { if mpUpload.Err != nil { return nil, mpUpload.Err } if objectName == mpUpload.Key { uploadIDs = append(uploadIDs, mpUpload.UploadID) } } // Return the latest upload id. return uploadIDs, nil } // listObjectPartsQuery (List Parts query) // - lists some or all (up to 1000) parts that have been uploaded // for a specific multipart upload // // You can use the request parameters as selection criteria to return // a subset of the uploads in a bucket, request parameters :- // --------- // ?part-number-marker - Specifies the part after which listing should // begin. // ?max-parts - Maximum parts to be listed per request. func (c *Client) listObjectPartsQuery(ctx context.Context, bucketName, objectName, uploadID string, partNumberMarker, maxParts int) (ListObjectPartsResult, error) { // Get resources properly escaped and lined up before using them in http request. urlValues := make(url.Values) // Set part number marker. urlValues.Set("part-number-marker", fmt.Sprintf("%d", partNumberMarker)) // Set upload id. urlValues.Set("uploadId", uploadID) // maxParts should be 1000 or less. if maxParts > 0 { // Set max parts. urlValues.Set("max-parts", fmt.Sprintf("%d", maxParts)) } // Execute GET on objectName to get list of parts. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return ListObjectPartsResult{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { return ListObjectPartsResult{}, httpRespToErrorResponse(resp, bucketName, objectName) } } // Decode list object parts XML. listObjectPartsResult := ListObjectPartsResult{} err = xmlDecoder(resp.Body, &listObjectPartsResult) if err != nil { return listObjectPartsResult, err } return listObjectPartsResult, nil } // Decode an S3 object name according to the encoding type func decodeS3Name(name, encodingType string) (string, error) { switch encodingType { case "url": return url.QueryUnescape(name) default: return name, nil } } minio-go-7.0.97/api-object-legal-hold.go000066400000000000000000000130121510244170000176750ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/xml" "fmt" "net/http" "net/url" "github.com/minio/minio-go/v7/pkg/s3utils" ) // objectLegalHold - object legal hold specified in // https://docs.aws.amazon.com/AmazonS3/latest/API/archive-RESTObjectPUTLegalHold.html type objectLegalHold struct { XMLNS string `xml:"xmlns,attr,omitempty"` XMLName xml.Name `xml:"LegalHold"` Status LegalHoldStatus `xml:"Status,omitempty"` } // PutObjectLegalHoldOptions represents options specified by user for PutObjectLegalHold call type PutObjectLegalHoldOptions struct { VersionID string Status *LegalHoldStatus } // GetObjectLegalHoldOptions represents options specified by user for GetObjectLegalHold call type GetObjectLegalHoldOptions struct { VersionID string } // LegalHoldStatus - object legal hold status. type LegalHoldStatus string const ( // LegalHoldEnabled indicates legal hold is enabled LegalHoldEnabled LegalHoldStatus = "ON" // LegalHoldDisabled indicates legal hold is disabled LegalHoldDisabled LegalHoldStatus = "OFF" ) func (r LegalHoldStatus) String() string { return string(r) } // IsValid - check whether this legal hold status is valid or not. func (r LegalHoldStatus) IsValid() bool { return r == LegalHoldEnabled || r == LegalHoldDisabled } func newObjectLegalHold(status *LegalHoldStatus) (*objectLegalHold, error) { if status == nil { return nil, fmt.Errorf("Status not set") } if !status.IsValid() { return nil, fmt.Errorf("invalid legal hold status `%v`", status) } legalHold := &objectLegalHold{ Status: *status, } return legalHold, nil } // PutObjectLegalHold sets the legal hold status for an object and specific version. // Legal hold prevents an object version from being overwritten or deleted, regardless of retention settings. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - objectName: Name of the object // - opts: Options including Status (LegalHoldEnabled or LegalHoldDisabled) and optional VersionID // // Returns an error if the operation fails or if the status is invalid. func (c *Client) PutObjectLegalHold(ctx context.Context, bucketName, objectName string, opts PutObjectLegalHoldOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("legal-hold", "") if opts.VersionID != "" { urlValues.Set("versionId", opts.VersionID) } lh, err := newObjectLegalHold(opts.Status) if err != nil { return err } lhData, err := xml.Marshal(lh) if err != nil { return err } reqMetadata := requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, contentBody: bytes.NewReader(lhData), contentLength: int64(len(lhData)), contentMD5Base64: sumMD5Base64(lhData), contentSHA256Hex: sum256Hex(lhData), } // Execute PUT Object Legal Hold. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp != nil { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { return httpRespToErrorResponse(resp, bucketName, objectName) } } return nil } // GetObjectLegalHold retrieves the legal hold status for an object and specific version. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - objectName: Name of the object // - opts: Options including optional VersionID to target a specific version // // Returns the legal hold status (LegalHoldEnabled or LegalHoldDisabled) or an error if the operation fails. func (c *Client) GetObjectLegalHold(ctx context.Context, bucketName, objectName string, opts GetObjectLegalHoldOptions) (status *LegalHoldStatus, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } urlValues := make(url.Values) urlValues.Set("legal-hold", "") if opts.VersionID != "" { urlValues.Set("versionId", opts.VersionID) } // Execute GET on bucket to list objects. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return nil, err } if resp != nil { if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, bucketName, objectName) } } lh := &objectLegalHold{} if err = xml.NewDecoder(resp.Body).Decode(lh); err != nil { return nil, err } return &lh.Status, nil } minio-go-7.0.97/api-object-lock.go000066400000000000000000000155111510244170000166230ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/xml" "fmt" "net/http" "net/url" "time" "github.com/minio/minio-go/v7/pkg/s3utils" ) // RetentionMode - object retention mode. type RetentionMode string const ( // Governance - governance mode. Governance RetentionMode = "GOVERNANCE" // Compliance - compliance mode. Compliance RetentionMode = "COMPLIANCE" ) func (r RetentionMode) String() string { return string(r) } // IsValid - check whether this retention mode is valid or not. func (r RetentionMode) IsValid() bool { return r == Governance || r == Compliance } // ValidityUnit - retention validity unit. type ValidityUnit string const ( // Days - denotes no. of days. Days ValidityUnit = "DAYS" // Years - denotes no. of years. Years ValidityUnit = "YEARS" ) func (unit ValidityUnit) String() string { return string(unit) } // IsValid - check whether this validity unit is valid or not. func (unit ValidityUnit) isValid() bool { return unit == Days || unit == Years } // Retention - bucket level retention configuration. type Retention struct { Mode RetentionMode Validity time.Duration } func (r Retention) String() string { return fmt.Sprintf("{Mode:%v, Validity:%v}", r.Mode, r.Validity) } // IsEmpty - returns whether retention is empty or not. func (r Retention) IsEmpty() bool { return r.Mode == "" || r.Validity == 0 } // objectLockConfig - object lock configuration specified in // https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_ObjectLockConfiguration.html type objectLockConfig struct { XMLNS string `xml:"xmlns,attr,omitempty"` XMLName xml.Name `xml:"ObjectLockConfiguration"` ObjectLockEnabled string `xml:"ObjectLockEnabled"` Rule *struct { DefaultRetention struct { Mode RetentionMode `xml:"Mode"` Days *uint `xml:"Days"` Years *uint `xml:"Years"` } `xml:"DefaultRetention"` } `xml:"Rule,omitempty"` } func newObjectLockConfig(mode *RetentionMode, validity *uint, unit *ValidityUnit) (*objectLockConfig, error) { config := &objectLockConfig{ ObjectLockEnabled: "Enabled", } if mode != nil && validity != nil && unit != nil { if !mode.IsValid() { return nil, fmt.Errorf("invalid retention mode `%v`", mode) } if !unit.isValid() { return nil, fmt.Errorf("invalid validity unit `%v`", unit) } config.Rule = &struct { DefaultRetention struct { Mode RetentionMode `xml:"Mode"` Days *uint `xml:"Days"` Years *uint `xml:"Years"` } `xml:"DefaultRetention"` }{} config.Rule.DefaultRetention.Mode = *mode if *unit == Days { config.Rule.DefaultRetention.Days = validity } else { config.Rule.DefaultRetention.Years = validity } return config, nil } if mode == nil && validity == nil && unit == nil { return config, nil } return nil, fmt.Errorf("all of retention mode, validity and validity unit must be passed") } // SetBucketObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. func (c *Client) SetBucketObjectLockConfig(ctx context.Context, bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("object-lock", "") config, err := newObjectLockConfig(mode, validity, unit) if err != nil { return err } configData, err := xml.Marshal(config) if err != nil { return err } reqMetadata := requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentBody: bytes.NewReader(configData), contentLength: int64(len(configData)), contentMD5Base64: sumMD5Base64(configData), contentSHA256Hex: sum256Hex(configData), } // Execute PUT bucket object lock configuration. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp != nil { if resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucketName, "") } } return nil } // GetObjectLockConfig gets object lock configuration of given bucket. func (c *Client) GetObjectLockConfig(ctx context.Context, bucketName string) (objectLock string, mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", nil, nil, nil, err } urlValues := make(url.Values) urlValues.Set("object-lock", "") // Execute GET on bucket to list objects. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return "", nil, nil, nil, err } if resp != nil { if resp.StatusCode != http.StatusOK { return "", nil, nil, nil, httpRespToErrorResponse(resp, bucketName, "") } } config := &objectLockConfig{} if err = xml.NewDecoder(resp.Body).Decode(config); err != nil { return "", nil, nil, nil, err } if config.Rule != nil { mode = &config.Rule.DefaultRetention.Mode if config.Rule.DefaultRetention.Days != nil { validity = config.Rule.DefaultRetention.Days days := Days unit = &days } else { validity = config.Rule.DefaultRetention.Years years := Years unit = &years } return config.ObjectLockEnabled, mode, validity, unit, nil } return config.ObjectLockEnabled, nil, nil, nil, nil } // GetBucketObjectLockConfig gets object lock configuration of given bucket. func (c *Client) GetBucketObjectLockConfig(ctx context.Context, bucketName string) (mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) { _, mode, validity, unit, err = c.GetObjectLockConfig(ctx, bucketName) return mode, validity, unit, err } // SetObjectLockConfig sets object lock configuration in given bucket. mode, validity and unit are either all set or all nil. func (c *Client) SetObjectLockConfig(ctx context.Context, bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error { return c.SetBucketObjectLockConfig(ctx, bucketName, mode, validity, unit) } minio-go-7.0.97/api-object-retention.go000066400000000000000000000127561510244170000177120ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2019-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/xml" "fmt" "net/http" "net/url" "time" "github.com/minio/minio-go/v7/pkg/s3utils" ) // objectRetention - object retention specified in // https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_ObjectLockConfiguration.html type objectRetention struct { XMLNS string `xml:"xmlns,attr,omitempty"` XMLName xml.Name `xml:"Retention"` Mode RetentionMode `xml:"Mode,omitempty"` RetainUntilDate *time.Time `type:"timestamp" timestampFormat:"iso8601" xml:"RetainUntilDate,omitempty"` } func newObjectRetention(mode *RetentionMode, date *time.Time) (*objectRetention, error) { objectRetention := &objectRetention{} if date != nil && !date.IsZero() { objectRetention.RetainUntilDate = date } if mode != nil { if !mode.IsValid() { return nil, fmt.Errorf("invalid retention mode `%v`", mode) } objectRetention.Mode = *mode } return objectRetention, nil } // PutObjectRetentionOptions represents options specified by user for PutObject call type PutObjectRetentionOptions struct { GovernanceBypass bool Mode *RetentionMode RetainUntilDate *time.Time VersionID string } // PutObjectRetention sets the retention configuration for an object and specific version. // Object retention prevents an object version from being deleted or overwritten for a specified period. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - objectName: Name of the object // - opts: Options including Mode (GOVERNANCE or COMPLIANCE), RetainUntilDate, optional VersionID, and GovernanceBypass // // Returns an error if the operation fails or if the retention settings are invalid. func (c *Client) PutObjectRetention(ctx context.Context, bucketName, objectName string, opts PutObjectRetentionOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("retention", "") if opts.VersionID != "" { urlValues.Set("versionId", opts.VersionID) } retention, err := newObjectRetention(opts.Mode, opts.RetainUntilDate) if err != nil { return err } retentionData, err := xml.Marshal(retention) if err != nil { return err } // Build headers. headers := make(http.Header) if opts.GovernanceBypass { // Set the bypass goverenance retention header headers.Set(amzBypassGovernance, "true") } reqMetadata := requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, contentBody: bytes.NewReader(retentionData), contentLength: int64(len(retentionData)), contentMD5Base64: sumMD5Base64(retentionData), contentSHA256Hex: sum256Hex(retentionData), customHeader: headers, } // Execute PUT Object Retention. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp != nil { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { return httpRespToErrorResponse(resp, bucketName, objectName) } } return nil } // GetObjectRetention retrieves the retention configuration for an object and specific version. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - objectName: Name of the object // - versionID: Optional version ID to target a specific version (empty string for current version) // // Returns the retention mode (GOVERNANCE or COMPLIANCE), retain-until date, and any error. func (c *Client) GetObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, nil, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return nil, nil, err } urlValues := make(url.Values) urlValues.Set("retention", "") if versionID != "" { urlValues.Set("versionId", versionID) } // Execute GET on bucket to list objects. resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return nil, nil, err } if resp != nil { if resp.StatusCode != http.StatusOK { return nil, nil, httpRespToErrorResponse(resp, bucketName, objectName) } } retention := &objectRetention{} if err = xml.NewDecoder(resp.Body).Decode(retention); err != nil { return nil, nil, err } return &retention.Mode, retention.RetainUntilDate, nil } minio-go-7.0.97/api-object-tagging.go000066400000000000000000000140261510244170000173130ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/xml" "net/http" "net/url" "github.com/minio/minio-go/v7/pkg/s3utils" "github.com/minio/minio-go/v7/pkg/tags" ) // PutObjectTaggingOptions holds an object version id // to update tag(s) of a specific object version type PutObjectTaggingOptions struct { VersionID string Internal AdvancedObjectTaggingOptions } // AdvancedObjectTaggingOptions for internal use by MinIO server - not intended for client use. type AdvancedObjectTaggingOptions struct { ReplicationProxyRequest string } // PutObjectTagging replaces or creates object tag(s) and can target a specific object version // in a versioned bucket. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - objectName: Name of the object // - otags: Tags to apply to the object // - opts: Options including VersionID to target a specific version // // Returns an error if the operation fails. func (c *Client) PutObjectTagging(ctx context.Context, bucketName, objectName string, otags *tags.Tags, opts PutObjectTaggingOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("tagging", "") if opts.VersionID != "" { urlValues.Set("versionId", opts.VersionID) } headers := make(http.Header, 0) if opts.Internal.ReplicationProxyRequest != "" { headers.Set(minIOBucketReplicationProxyRequest, opts.Internal.ReplicationProxyRequest) } reqBytes, err := xml.Marshal(otags) if err != nil { return err } reqMetadata := requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, contentBody: bytes.NewReader(reqBytes), contentLength: int64(len(reqBytes)), contentMD5Base64: sumMD5Base64(reqBytes), customHeader: headers, } // Execute PUT to set a object tagging. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp != nil { if resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucketName, objectName) } } return nil } // GetObjectTaggingOptions holds the object version ID // to fetch the tagging key/value pairs type GetObjectTaggingOptions struct { VersionID string Internal AdvancedObjectTaggingOptions } // GetObjectTagging retrieves object tag(s) with options to target a specific object version // in a versioned bucket. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - objectName: Name of the object // - opts: Options including VersionID to target a specific version // // Returns the object's tags or an error if the operation fails. func (c *Client) GetObjectTagging(ctx context.Context, bucketName, objectName string, opts GetObjectTaggingOptions) (*tags.Tags, error) { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("tagging", "") if opts.VersionID != "" { urlValues.Set("versionId", opts.VersionID) } headers := make(http.Header, 0) if opts.Internal.ReplicationProxyRequest != "" { headers.Set(minIOBucketReplicationProxyRequest, opts.Internal.ReplicationProxyRequest) } // Execute GET on object to get object tag(s) resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, customHeader: headers, }) defer closeResponse(resp) if err != nil { return nil, err } if resp != nil { if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, bucketName, objectName) } } return tags.ParseObjectXML(resp.Body) } // RemoveObjectTaggingOptions holds the version id of the object to remove type RemoveObjectTaggingOptions struct { VersionID string Internal AdvancedObjectTaggingOptions } // RemoveObjectTagging removes object tag(s) with options to target a specific object version // in a versioned bucket. // // Parameters: // - ctx: Context for request cancellation and timeout // - bucketName: Name of the bucket // - objectName: Name of the object // - opts: Options including VersionID to target a specific version // // Returns an error if the operation fails. func (c *Client) RemoveObjectTagging(ctx context.Context, bucketName, objectName string, opts RemoveObjectTaggingOptions) error { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) urlValues.Set("tagging", "") if opts.VersionID != "" { urlValues.Set("versionId", opts.VersionID) } headers := make(http.Header, 0) if opts.Internal.ReplicationProxyRequest != "" { headers.Set(minIOBucketReplicationProxyRequest, opts.Internal.ReplicationProxyRequest) } // Execute DELETE on object to remove object tag(s) resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, customHeader: headers, }) defer closeResponse(resp) if err != nil { return err } if resp != nil { // S3 returns "204 No content" after Object tag deletion. if resp.StatusCode != http.StatusNoContent { return httpRespToErrorResponse(resp, bucketName, objectName) } } return err } minio-go-7.0.97/api-presigned.go000066400000000000000000000176501510244170000164150ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "errors" "net/http" "net/url" "time" "github.com/minio/minio-go/v7/pkg/s3utils" "github.com/minio/minio-go/v7/pkg/signer" ) // presignURL - Returns a presigned URL for an input 'method'. // Expires maximum is 7days - ie. 604800 and minimum is 1. func (c *Client) presignURL(ctx context.Context, method, bucketName, objectName string, expires time.Duration, reqParams url.Values, extraHeaders http.Header) (u *url.URL, err error) { // Input validation. if method == "" { return nil, errInvalidArgument("method cannot be empty.") } if err = s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err } if err = isValidExpiry(expires); err != nil { return nil, err } // Convert expires into seconds. expireSeconds := int64(expires / time.Second) reqMetadata := requestMetadata{ presignURL: true, bucketName: bucketName, objectName: objectName, expires: expireSeconds, queryValues: reqParams, extraPresignHeader: extraHeaders, } // Instantiate a new request. // Since expires is set newRequest will presign the request. var req *http.Request if req, err = c.newRequest(ctx, method, reqMetadata); err != nil { return nil, err } return req.URL, nil } // PresignedGetObject - Returns a presigned URL to access an object // data without credentials. URL can have a maximum expiry of // upto 7days or a minimum of 1sec. Additionally you can override // a set of response headers using the query parameters. func (c *Client) PresignedGetObject(ctx context.Context, bucketName, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } return c.presignURL(ctx, http.MethodGet, bucketName, objectName, expires, reqParams, nil) } // PresignedHeadObject - Returns a presigned URL to access // object metadata without credentials. URL can have a maximum expiry // of upto 7days or a minimum of 1sec. Additionally you can override // a set of response headers using the query parameters. func (c *Client) PresignedHeadObject(ctx context.Context, bucketName, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } return c.presignURL(ctx, http.MethodHead, bucketName, objectName, expires, reqParams, nil) } // PresignedPutObject - Returns a presigned URL to upload an object // without credentials. URL can have a maximum expiry of upto 7days // or a minimum of 1sec. func (c *Client) PresignedPutObject(ctx context.Context, bucketName, objectName string, expires time.Duration) (u *url.URL, err error) { if err = s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } return c.presignURL(ctx, http.MethodPut, bucketName, objectName, expires, nil, nil) } // PresignHeader - similar to Presign() but allows including HTTP headers that // will be used to build the signature. The request using the resulting URL will // need to have the exact same headers to be added for signature validation to // pass. // // FIXME: The extra header parameter should be included in Presign() in the next // major version bump, and this function should then be deprecated. func (c *Client) PresignHeader(ctx context.Context, method, bucketName, objectName string, expires time.Duration, reqParams url.Values, extraHeaders http.Header) (u *url.URL, err error) { return c.presignURL(ctx, method, bucketName, objectName, expires, reqParams, extraHeaders) } // Presign - returns a presigned URL for any http method of your choice along // with custom request params and extra signed headers. URL can have a maximum // expiry of upto 7days or a minimum of 1sec. func (c *Client) Presign(ctx context.Context, method, bucketName, objectName string, expires time.Duration, reqParams url.Values) (u *url.URL, err error) { return c.presignURL(ctx, method, bucketName, objectName, expires, reqParams, nil) } // PresignedPostPolicy - Returns POST urlString, form data to upload an object. func (c *Client) PresignedPostPolicy(ctx context.Context, p *PostPolicy) (u *url.URL, formData map[string]string, err error) { // Validate input arguments. if p.expiration.IsZero() { return nil, nil, errors.New("Expiration time must be specified") } if _, ok := p.formData["key"]; !ok { return nil, nil, errors.New("object key must be specified") } if _, ok := p.formData["bucket"]; !ok { return nil, nil, errors.New("bucket name must be specified") } bucketName := p.formData["bucket"] // Fetch the bucket location. location, err := c.getBucketLocation(ctx, bucketName) if err != nil { return nil, nil, err } isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, bucketName) u, err = c.makeTargetURL(bucketName, "", location, isVirtualHost, nil) if err != nil { return nil, nil, err } // Get credentials from the configured credentials provider. credValues, err := c.credsProvider.GetWithContext(c.CredContext()) if err != nil { return nil, nil, err } var ( signerType = credValues.SignerType sessionToken = credValues.SessionToken accessKeyID = credValues.AccessKeyID secretAccessKey = credValues.SecretAccessKey ) if signerType.IsAnonymous() { return nil, nil, errInvalidArgument("Presigned operations are not supported for anonymous credentials") } // Keep time. t := time.Now().UTC() // For signature version '2' handle here. if signerType.IsV2() { policyBase64 := p.base64() p.formData["policy"] = policyBase64 // For Google endpoint set this value to be 'GoogleAccessId'. if s3utils.IsGoogleEndpoint(*c.endpointURL) { p.formData["GoogleAccessId"] = accessKeyID } else { // For all other endpoints set this value to be 'AWSAccessKeyId'. p.formData["AWSAccessKeyId"] = accessKeyID } // Sign the policy. p.formData["signature"] = signer.PostPresignSignatureV2(policyBase64, secretAccessKey) return u, p.formData, nil } // Add date policy. if err = p.addNewPolicy(policyCondition{ matchType: "eq", condition: "$x-amz-date", value: t.Format(iso8601DateFormat), }); err != nil { return nil, nil, err } // Add algorithm policy. if err = p.addNewPolicy(policyCondition{ matchType: "eq", condition: "$x-amz-algorithm", value: signV4Algorithm, }); err != nil { return nil, nil, err } // Add a credential policy. credential := signer.GetCredential(accessKeyID, location, t, signer.ServiceTypeS3) if err = p.addNewPolicy(policyCondition{ matchType: "eq", condition: "$x-amz-credential", value: credential, }); err != nil { return nil, nil, err } if sessionToken != "" { if err = p.addNewPolicy(policyCondition{ matchType: "eq", condition: "$x-amz-security-token", value: sessionToken, }); err != nil { return nil, nil, err } } // Get base64 encoded policy. policyBase64 := p.base64() // Fill in the form data. p.formData["policy"] = policyBase64 p.formData["x-amz-algorithm"] = signV4Algorithm p.formData["x-amz-credential"] = credential p.formData["x-amz-date"] = t.Format(iso8601DateFormat) if sessionToken != "" { p.formData["x-amz-security-token"] = sessionToken } p.formData["x-amz-signature"] = signer.PostPresignSignatureV4(policyBase64, t, secretAccessKey, location) return u, p.formData, nil } minio-go-7.0.97/api-prompt-object.go000066400000000000000000000046201510244170000172130ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2024 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/json" "io" "net/http" "github.com/minio/minio-go/v7/pkg/s3utils" ) // PromptObject performs language model inference with the prompt and referenced object as context. // Inference is performed using a Lambda handler that can process the prompt and object. // Currently, this functionality is limited to certain MinIO servers. func (c *Client) PromptObject(ctx context.Context, bucketName, objectName, prompt string, opts PromptObjectOptions) (io.ReadCloser, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, ErrorResponse{ StatusCode: http.StatusBadRequest, Code: InvalidBucketName, Message: err.Error(), } } if err := s3utils.CheckValidObjectName(objectName); err != nil { return nil, ErrorResponse{ StatusCode: http.StatusBadRequest, Code: XMinioInvalidObjectName, Message: err.Error(), } } opts.AddLambdaArnToReqParams(opts.LambdaArn) opts.SetHeader("Content-Type", "application/json") opts.AddPromptArg("prompt", prompt) promptReqBytes, err := json.Marshal(opts.PromptArgs) if err != nil { return nil, err } // Execute POST on bucket/object. resp, err := c.executeMethod(ctx, http.MethodPost, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: opts.toQueryValues(), customHeader: opts.Header(), contentSHA256Hex: sum256Hex(promptReqBytes), contentBody: bytes.NewReader(promptReqBytes), contentLength: int64(len(promptReqBytes)), }) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { defer closeResponse(resp) return nil, httpRespToErrorResponse(resp, bucketName, objectName) } return resp.Body, nil } minio-go-7.0.97/api-prompt-options.go000066400000000000000000000050221510244170000174350ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2024 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "net/http" "net/url" ) // PromptObjectOptions provides options to PromptObject call. // LambdaArn is the ARN of the Prompt Lambda to be invoked. // PromptArgs is a map of key-value pairs to be passed to the inference action on the Prompt Lambda. // "prompt" is a reserved key and should not be used as a key in PromptArgs. type PromptObjectOptions struct { LambdaArn string PromptArgs map[string]any headers map[string]string reqParams url.Values } // Header returns the http.Header representation of the POST options. func (o PromptObjectOptions) Header() http.Header { headers := make(http.Header, len(o.headers)) for k, v := range o.headers { headers.Set(k, v) } return headers } // AddPromptArg Add a key value pair to the prompt arguments where the key is a string and // the value is a JSON serializable. func (o *PromptObjectOptions) AddPromptArg(key string, value any) { if o.PromptArgs == nil { o.PromptArgs = make(map[string]any) } o.PromptArgs[key] = value } // AddLambdaArnToReqParams adds the lambdaArn to the request query string parameters. func (o *PromptObjectOptions) AddLambdaArnToReqParams(lambdaArn string) { if o.reqParams == nil { o.reqParams = make(url.Values) } o.reqParams.Add("lambdaArn", lambdaArn) } // SetHeader adds a key value pair to the options. The // key-value pair will be part of the HTTP POST request // headers. func (o *PromptObjectOptions) SetHeader(key, value string) { if o.headers == nil { o.headers = make(map[string]string) } o.headers[http.CanonicalHeaderKey(key)] = value } // toQueryValues - Convert the reqParams in Options to query string parameters. func (o *PromptObjectOptions) toQueryValues() url.Values { urlValues := make(url.Values) if o.reqParams != nil { for key, values := range o.reqParams { for _, value := range values { urlValues.Add(key, value) } } } return urlValues } minio-go-7.0.97/api-put-bucket.go000066400000000000000000000076561510244170000165250ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/xml" "net/http" "github.com/minio/minio-go/v7/pkg/s3utils" ) // Bucket operations func (c *Client) makeBucket(ctx context.Context, bucketName string, opts MakeBucketOptions) (err error) { // Validate the input arguments. if err := s3utils.CheckValidBucketNameStrict(bucketName); err != nil { return err } err = c.doMakeBucket(ctx, bucketName, opts) if err != nil && (opts.Region == "" || opts.Region == "us-east-1") { if resp, ok := err.(ErrorResponse); ok && resp.Code == AuthorizationHeaderMalformed && resp.Region != "" { opts.Region = resp.Region err = c.doMakeBucket(ctx, bucketName, opts) } } return err } func (c *Client) doMakeBucket(ctx context.Context, bucketName string, opts MakeBucketOptions) (err error) { defer func() { // Save the location into cache on a successful makeBucket response. if err == nil { c.bucketLocCache.Set(bucketName, opts.Region) } }() // If location is empty, treat is a default region 'us-east-1'. if opts.Region == "" { opts.Region = "us-east-1" // For custom region clients, default // to custom region instead not 'us-east-1'. if c.region != "" { opts.Region = c.region } } // PUT bucket request metadata. reqMetadata := requestMetadata{ bucketName: bucketName, bucketLocation: opts.Region, } headers := make(http.Header) if opts.ObjectLocking { headers.Add("x-amz-bucket-object-lock-enabled", "true") } if opts.ForceCreate { headers.Add("x-minio-force-create", "true") } reqMetadata.customHeader = headers // If location is not 'us-east-1' create bucket location config. if opts.Region != "us-east-1" && opts.Region != "" { createBucketConfig := createBucketConfiguration{} createBucketConfig.Location = opts.Region var createBucketConfigBytes []byte createBucketConfigBytes, err = xml.Marshal(createBucketConfig) if err != nil { return err } reqMetadata.contentMD5Base64 = sumMD5Base64(createBucketConfigBytes) reqMetadata.contentSHA256Hex = sum256Hex(createBucketConfigBytes) reqMetadata.contentBody = bytes.NewReader(createBucketConfigBytes) reqMetadata.contentLength = int64(len(createBucketConfigBytes)) } // Execute PUT to create a new bucket. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return err } if resp != nil { if resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucketName, "") } } // Success. return nil } // MakeBucketOptions holds all options to tweak bucket creation type MakeBucketOptions struct { // Bucket location Region string // Enable object locking ObjectLocking bool // ForceCreate - this is a MinIO specific extension. ForceCreate bool } // MakeBucket creates a new bucket with bucketName with a context to control cancellations and timeouts. // // Location is an optional argument, by default all buckets are // created in US Standard Region. // // For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html // For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations func (c *Client) MakeBucket(ctx context.Context, bucketName string, opts MakeBucketOptions) (err error) { return c.makeBucket(ctx, bucketName, opts) } minio-go-7.0.97/api-put-object-common.go000066400000000000000000000110441510244170000177660ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "io" "math" "os" "github.com/minio/minio-go/v7/pkg/s3utils" ) const nullVersionID = "null" // Verify if reader is *minio.Object func isObject(reader io.Reader) (ok bool) { _, ok = reader.(*Object) return ok } // Verify if reader is a generic ReaderAt func isReadAt(reader io.Reader) (ok bool) { var v *os.File v, ok = reader.(*os.File) if ok { // Stdin, Stdout and Stderr all have *os.File type // which happen to also be io.ReaderAt compatible // we need to add special conditions for them to // be ignored by this function. for _, f := range []string{ "/dev/stdin", "/dev/stdout", "/dev/stderr", } { if f == v.Name() { ok = false break } } } else { _, ok = reader.(io.ReaderAt) } return ok } // OptimalPartInfo - calculate the optimal part info for a given // object size. // // NOTE: Assumption here is that for any object to be uploaded to any S3 compatible // object storage it will have the following parameters as constants. // // maxPartsCount - 10000 // minPartSize - 16MiB // maxMultipartPutObjectSize - 5TiB func OptimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize, lastPartSize int64, err error) { // object size is '-1' set it to 5TiB. var unknownSize bool if objectSize == -1 { unknownSize = true objectSize = maxMultipartPutObjectSize } // object size is larger than supported maximum. if objectSize > maxMultipartPutObjectSize { err = errEntityTooLarge(objectSize, maxMultipartPutObjectSize, "", "") return totalPartsCount, partSize, lastPartSize, err } var partSizeFlt float64 if configuredPartSize > 0 { if int64(configuredPartSize) > objectSize { err = errEntityTooLarge(int64(configuredPartSize), objectSize, "", "") return totalPartsCount, partSize, lastPartSize, err } if !unknownSize { if objectSize > (int64(configuredPartSize) * maxPartsCount) { err = errInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.") return totalPartsCount, partSize, lastPartSize, err } } if configuredPartSize < absMinPartSize { err = errInvalidArgument("Input part size is smaller than allowed minimum of 5MiB.") return totalPartsCount, partSize, lastPartSize, err } if configuredPartSize > maxPartSize { err = errInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.") return totalPartsCount, partSize, lastPartSize, err } partSizeFlt = float64(configuredPartSize) if unknownSize { // If input has unknown size and part size is configured // keep it to maximum allowed as per 10000 parts. objectSize = int64(configuredPartSize) * maxPartsCount } } else { configuredPartSize = minPartSize // Use floats for part size for all calculations to avoid // overflows during float64 to int64 conversions. partSizeFlt = float64(objectSize / maxPartsCount) partSizeFlt = math.Ceil(partSizeFlt/float64(configuredPartSize)) * float64(configuredPartSize) } // Total parts count. totalPartsCount = int(math.Ceil(float64(objectSize) / partSizeFlt)) // Part size. partSize = int64(partSizeFlt) // Last part size. lastPartSize = objectSize - int64(totalPartsCount-1)*partSize return totalPartsCount, partSize, lastPartSize, nil } // getUploadID - fetch upload id if already present for an object name // or initiate a new request to fetch a new upload id. func (c *Client) newUploadID(ctx context.Context, bucketName, objectName string, opts PutObjectOptions) (uploadID string, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return "", err } // Initiate multipart upload for an object. initMultipartUploadResult, err := c.initiateMultipartUpload(ctx, bucketName, objectName, opts) if err != nil { return "", err } return initMultipartUploadResult.UploadID, nil } minio-go-7.0.97/api-put-object-fan-out.go000066400000000000000000000115271510244170000200550ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2023 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "encoding/json" "errors" "io" "mime/multipart" "net/http" "strconv" "strings" "time" "github.com/minio/minio-go/v7/pkg/encrypt" ) // PutObjectFanOutEntry is per object entry fan-out metadata type PutObjectFanOutEntry struct { Key string `json:"key"` UserMetadata map[string]string `json:"metadata,omitempty"` UserTags map[string]string `json:"tags,omitempty"` ContentType string `json:"contentType,omitempty"` ContentEncoding string `json:"contentEncoding,omitempty"` ContentDisposition string `json:"contentDisposition,omitempty"` ContentLanguage string `json:"contentLanguage,omitempty"` CacheControl string `json:"cacheControl,omitempty"` Retention RetentionMode `json:"retention,omitempty"` RetainUntilDate *time.Time `json:"retainUntil,omitempty"` } // PutObjectFanOutRequest this is the request structure sent // to the server to fan-out the stream to multiple objects. type PutObjectFanOutRequest struct { Entries []PutObjectFanOutEntry Checksum Checksum SSE encrypt.ServerSide } // PutObjectFanOutResponse this is the response structure sent // by the server upon success or failure for each object // fan-out keys. Additionally, this response carries ETag, // VersionID and LastModified for each object fan-out. type PutObjectFanOutResponse struct { Key string `json:"key"` ETag string `json:"etag,omitempty"` VersionID string `json:"versionId,omitempty"` LastModified *time.Time `json:"lastModified,omitempty"` Error string `json:"error,omitempty"` } // PutObjectFanOut - is a variant of PutObject instead of writing a single object from a single // stream multiple objects are written, defined via a list of PutObjectFanOutRequests. Each entry // in PutObjectFanOutRequest carries an object keyname and its relevant metadata if any. `Key` is // mandatory, rest of the other options in PutObjectFanOutRequest are optional. func (c *Client) PutObjectFanOut(ctx context.Context, bucket string, fanOutData io.Reader, fanOutReq PutObjectFanOutRequest) ([]PutObjectFanOutResponse, error) { if len(fanOutReq.Entries) == 0 { return nil, errInvalidArgument("fan out requests cannot be empty") } policy := NewPostPolicy() policy.SetBucket(bucket) policy.SetKey(strconv.FormatInt(time.Now().UnixNano(), 16)) // Expires in 15 minutes. policy.SetExpires(time.Now().UTC().Add(15 * time.Minute)) // Set encryption headers if any. policy.SetEncryption(fanOutReq.SSE) // Set checksum headers if any. err := policy.SetChecksum(fanOutReq.Checksum) if err != nil { return nil, err } url, formData, err := c.PresignedPostPolicy(ctx, policy) if err != nil { return nil, err } r, w := io.Pipe() req, err := http.NewRequest(http.MethodPost, url.String(), r) if err != nil { w.Close() return nil, err } var b strings.Builder enc := json.NewEncoder(&b) for _, req := range fanOutReq.Entries { if req.Key == "" { w.Close() return nil, errors.New("PutObjectFanOutRequest.Key is mandatory and cannot be empty") } if err = enc.Encode(&req); err != nil { w.Close() return nil, err } } mwriter := multipart.NewWriter(w) req.Header.Add("Content-Type", mwriter.FormDataContentType()) go func() { defer w.Close() defer mwriter.Close() for k, v := range formData { if err := mwriter.WriteField(k, v); err != nil { return } } if err := mwriter.WriteField("x-minio-fanout-list", b.String()); err != nil { return } mw, err := mwriter.CreateFormFile("file", "fanout-content") if err != nil { return } if _, err = io.Copy(mw, fanOutData); err != nil { return } }() resp, err := c.do(req) if err != nil { return nil, err } defer closeResponse(resp) if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, bucket, "fanout-content") } dec := json.NewDecoder(resp.Body) fanOutResp := make([]PutObjectFanOutResponse, 0, len(fanOutReq.Entries)) for dec.More() { var m PutObjectFanOutResponse if err = dec.Decode(&m); err != nil { return nil, err } fanOutResp = append(fanOutResp, m) } return fanOutResp, nil } minio-go-7.0.97/api-put-object-file-context.go000066400000000000000000000037101510244170000211000ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "mime" "os" "path/filepath" "github.com/minio/minio-go/v7/pkg/s3utils" ) // FPutObject - Create an object in a bucket, with contents from file at filePath. Allows request cancellation. func (c *Client) FPutObject(ctx context.Context, bucketName, objectName, filePath string, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return UploadInfo{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return UploadInfo{}, err } // Open the referenced file. fileReader, err := os.Open(filePath) // If any error fail quickly here. if err != nil { return UploadInfo{}, err } defer fileReader.Close() // Save the file stat. fileStat, err := fileReader.Stat() if err != nil { return UploadInfo{}, err } // Save the file size. fileSize := fileStat.Size() // Set contentType based on filepath extension if not given or default // value of "application/octet-stream" if the extension has no associated type. if opts.ContentType == "" { if opts.ContentType = mime.TypeByExtension(filepath.Ext(filePath)); opts.ContentType == "" { opts.ContentType = "application/octet-stream" } } return c.PutObject(ctx, bucketName, objectName, fileReader, fileSize, opts) } minio-go-7.0.97/api-put-object-multipart.go000066400000000000000000000361531510244170000205270ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/base64" "encoding/hex" "encoding/xml" "fmt" "io" "net/http" "net/url" "sort" "strconv" "strings" "github.com/google/uuid" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/s3utils" ) func (c *Client) putObjectMultipart(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions, ) (info UploadInfo, err error) { info, err = c.putObjectMultipartNoStream(ctx, bucketName, objectName, reader, opts) if err != nil { errResp := ToErrorResponse(err) // Verify if multipart functionality is not available, if not // fall back to single PutObject operation. if errResp.Code == AccessDenied && strings.Contains(errResp.Message, "Access Denied") { // Verify if size of reader is greater than '5GiB'. if size > maxSinglePutObjectSize { return UploadInfo{}, errEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) } // Fall back to uploading as single PutObject operation. return c.putObject(ctx, bucketName, objectName, reader, size, opts) } } return info, err } func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, objectName string, reader io.Reader, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { return UploadInfo{}, err } if err = s3utils.CheckValidObjectName(objectName); err != nil { return UploadInfo{}, err } // Total data read and written to server. should be equal to // 'size' at the end of the call. var totalUploadedSize int64 // Complete multipart upload. var complMultipartUpload completeMultipartUpload // Calculate the optimal parts info for a given size. totalPartsCount, partSize, _, err := OptimalPartInfo(-1, opts.PartSize) if err != nil { return UploadInfo{}, err } // Choose hash algorithms to be calculated by hashCopyN, // avoid sha256 with non-v4 signature request or // HTTPS connection. hashAlgos, hashSums := c.hashMaterials(opts.SendContentMd5, !opts.DisableContentSha256) // Initiate a new multipart upload. uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) if err != nil { return UploadInfo{}, err } defer func() { if err != nil { c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) } }() // Part number always starts with '1'. partNumber := 1 // Initialize parts uploaded map. partsInfo := make(map[int]ObjectPart) // Create a buffer. buf := make([]byte, partSize) // Create checksums // CRC32C is ~50% faster on AMD64 @ 30GB/s customHeader := make(http.Header) crc := opts.AutoChecksum.Hasher() for partNumber <= totalPartsCount { length, rErr := readFull(reader, buf) if rErr == io.EOF && partNumber > 1 { break } if rErr != nil && rErr != io.ErrUnexpectedEOF && rErr != io.EOF { return UploadInfo{}, rErr } // Calculates hash sums while copying partSize bytes into cw. for k, v := range hashAlgos { v.Write(buf[:length]) hashSums[k] = v.Sum(nil) v.Close() } // Update progress reader appropriately to the latest offset // as we read from the source. rd := newHook(bytes.NewReader(buf[:length]), opts.Progress) // Checksums.. var ( md5Base64 string sha256Hex string ) if hashSums["md5"] != nil { md5Base64 = base64.StdEncoding.EncodeToString(hashSums["md5"]) } if hashSums["sha256"] != nil { sha256Hex = hex.EncodeToString(hashSums["sha256"]) } if opts.AutoChecksum.IsSet() { crc.Reset() crc.Write(buf[:length]) cSum := crc.Sum(nil) customHeader.Set(opts.AutoChecksum.Key(), base64.StdEncoding.EncodeToString(cSum)) customHeader.Set(amzChecksumAlgo, opts.AutoChecksum.String()) if opts.AutoChecksum.FullObjectRequested() { customHeader.Set(amzChecksumMode, ChecksumFullObjectMode.String()) } } p := uploadPartParams{bucketName: bucketName, objectName: objectName, uploadID: uploadID, reader: rd, partNumber: partNumber, md5Base64: md5Base64, sha256Hex: sha256Hex, size: int64(length), sse: opts.ServerSideEncryption, streamSha256: !opts.DisableContentSha256, customHeader: customHeader} // Proceed to upload the part. objPart, uerr := c.uploadPart(ctx, p) if uerr != nil { return UploadInfo{}, uerr } // Save successfully uploaded part metadata. partsInfo[partNumber] = objPart // Save successfully uploaded size. totalUploadedSize += int64(length) // Increment part number. partNumber++ // For unknown size, Read EOF we break away. // We do not have to upload till totalPartsCount. if rErr == io.EOF { break } } // Loop over total uploaded parts to save them in // Parts array before completing the multipart request. allParts := make([]ObjectPart, 0, len(partsInfo)) for i := 1; i < partNumber; i++ { part, ok := partsInfo[i] if !ok { return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } allParts = append(allParts, part) complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, PartNumber: part.PartNumber, ChecksumCRC32: part.ChecksumCRC32, ChecksumCRC32C: part.ChecksumCRC32C, ChecksumSHA1: part.ChecksumSHA1, ChecksumSHA256: part.ChecksumSHA256, ChecksumCRC64NVME: part.ChecksumCRC64NVME, }) } // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) opts = PutObjectOptions{ ServerSideEncryption: opts.ServerSideEncryption, AutoChecksum: opts.AutoChecksum, } applyAutoChecksum(&opts, allParts) uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts) if err != nil { return UploadInfo{}, err } uploadInfo.Size = totalUploadedSize return uploadInfo, nil } // initiateMultipartUpload - Initiates a multipart upload and returns an upload ID. func (c *Client) initiateMultipartUpload(ctx context.Context, bucketName, objectName string, opts PutObjectOptions) (initiateMultipartUploadResult, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return initiateMultipartUploadResult{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return initiateMultipartUploadResult{}, err } // Initialize url queries. urlValues := make(url.Values) urlValues.Set("uploads", "") if opts.Internal.SourceVersionID != "" { if opts.Internal.SourceVersionID != nullVersionID { if _, err := uuid.Parse(opts.Internal.SourceVersionID); err != nil { return initiateMultipartUploadResult{}, errInvalidArgument(err.Error()) } } urlValues.Set("versionId", opts.Internal.SourceVersionID) } // Set ContentType header. customHeader := opts.Header() reqMetadata := requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, customHeader: customHeader, } // Execute POST on an objectName to initiate multipart upload. resp, err := c.executeMethod(ctx, http.MethodPost, reqMetadata) defer closeResponse(resp) if err != nil { return initiateMultipartUploadResult{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { return initiateMultipartUploadResult{}, httpRespToErrorResponse(resp, bucketName, objectName) } } // Decode xml for new multipart upload. initiateMultipartUploadResult := initiateMultipartUploadResult{} err = xmlDecoder(resp.Body, &initiateMultipartUploadResult) if err != nil { return initiateMultipartUploadResult, err } return initiateMultipartUploadResult, nil } type uploadPartParams struct { bucketName string objectName string uploadID string reader io.Reader partNumber int md5Base64 string sha256Hex string size int64 sse encrypt.ServerSide streamSha256 bool customHeader http.Header trailer http.Header } // uploadPart - Uploads a part in a multipart upload. func (c *Client) uploadPart(ctx context.Context, p uploadPartParams) (ObjectPart, error) { // Input validation. if err := s3utils.CheckValidBucketName(p.bucketName); err != nil { return ObjectPart{}, err } if err := s3utils.CheckValidObjectName(p.objectName); err != nil { return ObjectPart{}, err } if p.size > maxPartSize { return ObjectPart{}, errEntityTooLarge(p.size, maxPartSize, p.bucketName, p.objectName) } if p.size <= -1 { return ObjectPart{}, errEntityTooSmall(p.size, p.bucketName, p.objectName) } if p.partNumber <= 0 { return ObjectPart{}, errInvalidArgument("Part number cannot be negative or equal to zero.") } if p.uploadID == "" { return ObjectPart{}, errInvalidArgument("UploadID cannot be empty.") } // Get resources properly escaped and lined up before using them in http request. urlValues := make(url.Values) // Set part number. urlValues.Set("partNumber", strconv.Itoa(p.partNumber)) // Set upload id. urlValues.Set("uploadId", p.uploadID) // Set encryption headers, if any. if p.customHeader == nil { p.customHeader = make(http.Header) } // https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html // Server-side encryption is supported by the S3 Multipart Upload actions. // Unless you are using a customer-provided encryption key, you don't need // to specify the encryption parameters in each UploadPart request. if p.sse != nil && p.sse.Type() == encrypt.SSEC { p.sse.Marshal(p.customHeader) } reqMetadata := requestMetadata{ bucketName: p.bucketName, objectName: p.objectName, queryValues: urlValues, customHeader: p.customHeader, contentBody: p.reader, contentLength: p.size, contentMD5Base64: p.md5Base64, contentSHA256Hex: p.sha256Hex, streamSha256: p.streamSha256, trailer: p.trailer, } // Execute PUT on each part. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return ObjectPart{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { return ObjectPart{}, httpRespToErrorResponse(resp, p.bucketName, p.objectName) } } // Once successfully uploaded, return completed part. h := resp.Header objPart := ObjectPart{ ChecksumCRC32: h.Get(ChecksumCRC32.Key()), ChecksumCRC32C: h.Get(ChecksumCRC32C.Key()), ChecksumSHA1: h.Get(ChecksumSHA1.Key()), ChecksumSHA256: h.Get(ChecksumSHA256.Key()), ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()), } objPart.Size = p.size objPart.PartNumber = p.partNumber // Trim off the odd double quotes from ETag in the beginning and end. objPart.ETag = trimEtag(h.Get("ETag")) return objPart, nil } // completeMultipartUpload - Completes a multipart upload by assembling previously uploaded parts. func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string, complete completeMultipartUpload, opts PutObjectOptions, ) (UploadInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return UploadInfo{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return UploadInfo{}, err } // Initialize url queries. urlValues := make(url.Values) urlValues.Set("uploadId", uploadID) // Marshal complete multipart body. completeMultipartUploadBytes, err := xml.Marshal(complete) if err != nil { return UploadInfo{}, err } headers := opts.Header() if s3utils.IsAmazonEndpoint(*c.endpointURL) { headers.Del(encrypt.SseKmsKeyID) // Remove X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id not supported in CompleteMultipartUpload headers.Del(encrypt.SseGenericHeader) // Remove X-Amz-Server-Side-Encryption not supported in CompleteMultipartUpload headers.Del(encrypt.SseEncryptionContext) // Remove X-Amz-Server-Side-Encryption-Context not supported in CompleteMultipartUpload } // Instantiate all the complete multipart buffer. completeMultipartUploadBuffer := bytes.NewReader(completeMultipartUploadBytes) reqMetadata := requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, contentBody: completeMultipartUploadBuffer, contentLength: int64(len(completeMultipartUploadBytes)), contentSHA256Hex: sum256Hex(completeMultipartUploadBytes), customHeader: headers, expect200OKWithError: true, } // Execute POST to complete multipart upload for an objectName. resp, err := c.executeMethod(ctx, http.MethodPost, reqMetadata) defer closeResponse(resp) if err != nil { return UploadInfo{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { return UploadInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) } } // Read resp.Body into a []bytes to parse for Error response inside the body var b []byte b, err = io.ReadAll(resp.Body) if err != nil { return UploadInfo{}, err } // Decode completed multipart upload response on success. completeMultipartUploadResult := completeMultipartUploadResult{} err = xmlDecoder(bytes.NewReader(b), &completeMultipartUploadResult) if err != nil { // xml parsing failure due to presence an ill-formed xml fragment return UploadInfo{}, err } else if completeMultipartUploadResult.Bucket == "" { // xml's Decode method ignores well-formed xml that don't apply to the type of value supplied. // In this case, it would leave completeMultipartUploadResult with the corresponding zero-values // of the members. // Decode completed multipart upload response on failure completeMultipartUploadErr := ErrorResponse{} err = xmlDecoder(bytes.NewReader(b), &completeMultipartUploadErr) if err != nil { // xml parsing failure due to presence an ill-formed xml fragment return UploadInfo{}, err } return UploadInfo{}, completeMultipartUploadErr } // extract lifecycle expiry date and rule ID expTime, ruleID := amzExpirationToExpiryDateRuleID(resp.Header.Get(amzExpiration)) return UploadInfo{ Bucket: completeMultipartUploadResult.Bucket, Key: completeMultipartUploadResult.Key, ETag: trimEtag(completeMultipartUploadResult.ETag), VersionID: resp.Header.Get(amzVersionID), Location: completeMultipartUploadResult.Location, Expiration: expTime, ExpirationRuleID: ruleID, ChecksumSHA256: completeMultipartUploadResult.ChecksumSHA256, ChecksumSHA1: completeMultipartUploadResult.ChecksumSHA1, ChecksumCRC32: completeMultipartUploadResult.ChecksumCRC32, ChecksumCRC32C: completeMultipartUploadResult.ChecksumCRC32C, ChecksumCRC64NVME: completeMultipartUploadResult.ChecksumCRC64NVME, ChecksumMode: completeMultipartUploadResult.ChecksumType, }, nil } minio-go-7.0.97/api-put-object-streaming.go000066400000000000000000000610141510244170000204710ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/base64" "fmt" "io" "net/http" "net/url" "sort" "strings" "sync" "github.com/google/uuid" "github.com/minio/minio-go/v7/pkg/s3utils" ) // putObjectMultipartStream - upload a large object using // multipart upload and streaming signature for signing payload. // Comprehensive put object operation involving multipart uploads. // // Following code handles these types of readers. // // - *minio.Object // - Any reader which has a method 'ReadAt()' func (c *Client) putObjectMultipartStream(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions, ) (info UploadInfo, err error) { if opts.ConcurrentStreamParts && opts.NumThreads > 1 { info, err = c.putObjectMultipartStreamParallel(ctx, bucketName, objectName, reader, opts) } else if !isObject(reader) && isReadAt(reader) && !opts.SendContentMd5 { // Verify if the reader implements ReadAt and it is not a *minio.Object then we will use parallel uploader. info, err = c.putObjectMultipartStreamFromReadAt(ctx, bucketName, objectName, reader.(io.ReaderAt), size, opts) } else { info, err = c.putObjectMultipartStreamOptionalChecksum(ctx, bucketName, objectName, reader, size, opts) } if err != nil && s3utils.IsGoogleEndpoint(*c.endpointURL) { errResp := ToErrorResponse(err) // Verify if multipart functionality is not available, if not // fall back to single PutObject operation. if errResp.Code == AccessDenied && strings.Contains(errResp.Message, "Access Denied") { // Verify if size of reader is greater than '5GiB'. if size > maxSinglePutObjectSize { return UploadInfo{}, errEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) } // Fall back to uploading as single PutObject operation. return c.putObject(ctx, bucketName, objectName, reader, size, opts) } } return info, err } // uploadedPartRes - the response received from a part upload. type uploadedPartRes struct { Error error // Any error encountered while uploading the part. PartNum int // Number of the part uploaded. Size int64 // Size of the part uploaded. Part ObjectPart } type uploadPartReq struct { PartNum int // Number of the part uploaded. Part ObjectPart // Size of the part uploaded. } // putObjectMultipartFromReadAt - Uploads files bigger than 128MiB. // Supports all readers which implements io.ReaderAt interface // (ReadAt method). // // NOTE: This function is meant to be used for all readers which // implement io.ReaderAt which allows us for resuming multipart // uploads but reading at an offset, which would avoid re-read the // data which was already uploaded. Internally this function uses // temporary files for staging all the data, these temporary files are // cleaned automatically when the caller i.e http client closes the // stream after uploading all the contents successfully. func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketName, objectName string, reader io.ReaderAt, size int64, opts PutObjectOptions, ) (info UploadInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { return UploadInfo{}, err } if err = s3utils.CheckValidObjectName(objectName); err != nil { return UploadInfo{}, err } // Calculate the optimal parts info for a given size. totalPartsCount, partSize, lastPartSize, err := OptimalPartInfo(size, opts.PartSize) if err != nil { return UploadInfo{}, err } // Initiate a new multipart upload. uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) if err != nil { return UploadInfo{}, err } withChecksum := c.trailingHeaderSupport // Aborts the multipart upload in progress, if the // function returns any error, since we do not resume // we should purge the parts which have been uploaded // to relinquish storage space. defer func() { if err != nil { c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) } }() // Total data read and written to server. should be equal to 'size' at the end of the call. var totalUploadedSize int64 // Complete multipart upload. var complMultipartUpload completeMultipartUpload // Declare a channel that sends the next part number to be uploaded. uploadPartsCh := make(chan uploadPartReq) // Declare a channel that sends back the response of a part upload. uploadedPartsCh := make(chan uploadedPartRes) // Used for readability, lastPartNumber is always totalPartsCount. lastPartNumber := totalPartsCount partitionCtx, partitionCancel := context.WithCancel(ctx) defer partitionCancel() // Send each part number to the channel to be processed. go func() { defer close(uploadPartsCh) for p := 1; p <= totalPartsCount; p++ { select { case <-partitionCtx.Done(): return case uploadPartsCh <- uploadPartReq{PartNum: p}: } } }() // Receive each part number from the channel allowing three parallel uploads. for w := 1; w <= opts.getNumThreads(); w++ { go func(partSize int64) { for { var uploadReq uploadPartReq var ok bool select { case <-ctx.Done(): return case uploadReq, ok = <-uploadPartsCh: if !ok { return } // Each worker will draw from the part channel and upload in parallel. } // If partNumber was not uploaded we calculate the missing // part offset and size. For all other part numbers we // calculate offset based on multiples of partSize. readOffset := int64(uploadReq.PartNum-1) * partSize // As a special case if partNumber is lastPartNumber, we // calculate the offset based on the last part size. if uploadReq.PartNum == lastPartNumber { readOffset = size - lastPartSize partSize = lastPartSize } sectionReader := newHook(io.NewSectionReader(reader, readOffset, partSize), opts.Progress) trailer := make(http.Header, 1) if withChecksum { crc := opts.AutoChecksum.Hasher() trailer.Set(opts.AutoChecksum.Key(), base64.StdEncoding.EncodeToString(crc.Sum(nil))) sectionReader = newHashReaderWrapper(sectionReader, crc, func(hash []byte) { trailer.Set(opts.AutoChecksum.Key(), base64.StdEncoding.EncodeToString(hash)) }) } // Proceed to upload the part. p := uploadPartParams{ bucketName: bucketName, objectName: objectName, uploadID: uploadID, reader: sectionReader, partNumber: uploadReq.PartNum, size: partSize, sse: opts.ServerSideEncryption, streamSha256: !opts.DisableContentSha256, sha256Hex: "", trailer: trailer, } objPart, err := c.uploadPart(ctx, p) if err != nil { select { case <-ctx.Done(): case uploadedPartsCh <- uploadedPartRes{ Error: err, }: } // Exit the goroutine. return } // Save successfully uploaded part metadata. uploadReq.Part = objPart // Send successful part info through the channel. select { case <-ctx.Done(): case uploadedPartsCh <- uploadedPartRes{ Size: objPart.Size, PartNum: uploadReq.PartNum, Part: uploadReq.Part, }: } } }(partSize) } // Gather the responses as they occur and update any // progress bar. allParts := make([]ObjectPart, 0, totalPartsCount) for u := 1; u <= totalPartsCount; u++ { select { case <-ctx.Done(): return UploadInfo{}, ctx.Err() case uploadRes := <-uploadedPartsCh: if uploadRes.Error != nil { return UploadInfo{}, uploadRes.Error } allParts = append(allParts, uploadRes.Part) // Update the totalUploadedSize. totalUploadedSize += uploadRes.Size complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: uploadRes.Part.ETag, PartNumber: uploadRes.Part.PartNumber, ChecksumCRC32: uploadRes.Part.ChecksumCRC32, ChecksumCRC32C: uploadRes.Part.ChecksumCRC32C, ChecksumSHA1: uploadRes.Part.ChecksumSHA1, ChecksumSHA256: uploadRes.Part.ChecksumSHA256, ChecksumCRC64NVME: uploadRes.Part.ChecksumCRC64NVME, }) } } // Verify if we uploaded all the data. if totalUploadedSize != size { return UploadInfo{}, errUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) } // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) opts = PutObjectOptions{ ServerSideEncryption: opts.ServerSideEncryption, AutoChecksum: opts.AutoChecksum, } if withChecksum { applyAutoChecksum(&opts, allParts) } uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts) if err != nil { return UploadInfo{}, err } uploadInfo.Size = totalUploadedSize return uploadInfo, nil } func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions, ) (info UploadInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { return UploadInfo{}, err } if err = s3utils.CheckValidObjectName(objectName); err != nil { return UploadInfo{}, err } // Calculate the optimal parts info for a given size. totalPartsCount, partSize, lastPartSize, err := OptimalPartInfo(size, opts.PartSize) if err != nil { return UploadInfo{}, err } // Initiates a new multipart request uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) if err != nil { return UploadInfo{}, err } // Aborts the multipart upload if the function returns // any error, since we do not resume we should purge // the parts which have been uploaded to relinquish // storage space. defer func() { if err != nil { c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) } }() // Create checksums // CRC32C is ~50% faster on AMD64 @ 30GB/s customHeader := make(http.Header) crc := opts.AutoChecksum.Hasher() md5Hash := c.md5Hasher() defer md5Hash.Close() // Total data read and written to server. should be equal to 'size' at the end of the call. var totalUploadedSize int64 // Initialize parts uploaded map. partsInfo := make(map[int]ObjectPart) // Create a buffer. buf := make([]byte, partSize) // Avoid declaring variables in the for loop var md5Base64 string // Part number always starts with '1'. var partNumber int for partNumber = 1; partNumber <= totalPartsCount; partNumber++ { // Proceed to upload the part. if partNumber == totalPartsCount { partSize = lastPartSize } length, rerr := readFull(reader, buf) if rerr == io.EOF && partNumber > 1 { break } if rerr != nil && rerr != io.ErrUnexpectedEOF && err != io.EOF { return UploadInfo{}, rerr } // Calculate md5sum. if opts.SendContentMd5 { md5Hash.Reset() md5Hash.Write(buf[:length]) md5Base64 = base64.StdEncoding.EncodeToString(md5Hash.Sum(nil)) } if opts.AutoChecksum.IsSet() { // Add CRC32C instead. crc.Reset() crc.Write(buf[:length]) cSum := crc.Sum(nil) customHeader.Set(opts.AutoChecksum.Key(), base64.StdEncoding.EncodeToString(cSum)) customHeader.Set(amzChecksumAlgo, opts.AutoChecksum.String()) if opts.AutoChecksum.FullObjectRequested() { customHeader.Set(amzChecksumMode, ChecksumFullObjectMode.String()) } } // Update progress reader appropriately to the latest offset // as we read from the source. hooked := newHook(bytes.NewReader(buf[:length]), opts.Progress) p := uploadPartParams{bucketName: bucketName, objectName: objectName, uploadID: uploadID, reader: hooked, partNumber: partNumber, md5Base64: md5Base64, size: partSize, sse: opts.ServerSideEncryption, streamSha256: !opts.DisableContentSha256, customHeader: customHeader} objPart, uerr := c.uploadPart(ctx, p) if uerr != nil { return UploadInfo{}, uerr } // Save successfully uploaded part metadata. partsInfo[partNumber] = objPart // Save successfully uploaded size. totalUploadedSize += partSize } // Verify if we uploaded all the data. if size > 0 { if totalUploadedSize != size { return UploadInfo{}, errUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) } } // Complete multipart upload. var complMultipartUpload completeMultipartUpload // Loop over total uploaded parts to save them in // Parts array before completing the multipart request. allParts := make([]ObjectPart, 0, len(partsInfo)) for i := 1; i < partNumber; i++ { part, ok := partsInfo[i] if !ok { return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } allParts = append(allParts, part) complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, PartNumber: part.PartNumber, ChecksumCRC32: part.ChecksumCRC32, ChecksumCRC32C: part.ChecksumCRC32C, ChecksumSHA1: part.ChecksumSHA1, ChecksumSHA256: part.ChecksumSHA256, ChecksumCRC64NVME: part.ChecksumCRC64NVME, }) } // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) opts = PutObjectOptions{ ServerSideEncryption: opts.ServerSideEncryption, AutoChecksum: opts.AutoChecksum, } applyAutoChecksum(&opts, allParts) uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts) if err != nil { return UploadInfo{}, err } uploadInfo.Size = totalUploadedSize return uploadInfo, nil } // putObjectMultipartStreamParallel uploads opts.NumThreads parts in parallel. // This is expected to take opts.PartSize * opts.NumThreads * (GOGC / 100) bytes of buffer. func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketName, objectName string, reader io.Reader, opts PutObjectOptions, ) (info UploadInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { return UploadInfo{}, err } if err = s3utils.CheckValidObjectName(objectName); err != nil { return UploadInfo{}, err } // Cancel all when an error occurs. ctx, cancel := context.WithCancel(ctx) defer cancel() // Calculate the optimal parts info for a given size. totalPartsCount, partSize, _, err := OptimalPartInfo(-1, opts.PartSize) if err != nil { return UploadInfo{}, err } // Initiates a new multipart request uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) if err != nil { return UploadInfo{}, err } // Aborts the multipart upload if the function returns // any error, since we do not resume we should purge // the parts which have been uploaded to relinquish // storage space. defer func() { if err != nil { c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) } }() // Create checksums // CRC32C is ~50% faster on AMD64 @ 30GB/s crc := opts.AutoChecksum.Hasher() // Total data read and written to server. should be equal to 'size' at the end of the call. var totalUploadedSize int64 // Initialize parts uploaded map. partsInfo := make(map[int]ObjectPart) // Create a buffer. nBuffers := int64(opts.NumThreads) bufs := make(chan []byte, nBuffers) all := make([]byte, nBuffers*partSize) for i := int64(0); i < nBuffers; i++ { bufs <- all[i*partSize : i*partSize+partSize] } var wg sync.WaitGroup var mu sync.Mutex errCh := make(chan error, opts.NumThreads) reader = newHook(reader, opts.Progress) // Part number always starts with '1'. var partNumber int for partNumber = 1; partNumber <= totalPartsCount; partNumber++ { // Proceed to upload the part. var buf []byte select { case buf = <-bufs: case err = <-errCh: cancel() wg.Wait() return UploadInfo{}, err } if int64(len(buf)) != partSize { return UploadInfo{}, fmt.Errorf("read buffer < %d than expected partSize: %d", len(buf), partSize) } length, rerr := readFull(reader, buf) if rerr == io.EOF && partNumber > 1 { // Done break } if rerr != nil && rerr != io.ErrUnexpectedEOF && err != io.EOF { cancel() wg.Wait() return UploadInfo{}, rerr } wg.Add(1) go func(partNumber int) { // Calculate md5sum. customHeader := make(http.Header) if opts.AutoChecksum.IsSet() { // Add Checksum instead. crc.Reset() crc.Write(buf[:length]) cSum := crc.Sum(nil) customHeader.Set(opts.AutoChecksum.Key(), base64.StdEncoding.EncodeToString(cSum)) customHeader.Set(amzChecksumAlgo, opts.AutoChecksum.String()) if opts.AutoChecksum.FullObjectRequested() { customHeader.Set(amzChecksumMode, ChecksumFullObjectMode.String()) } } // Avoid declaring variables in the for loop var md5Base64 string if opts.SendContentMd5 { md5Hash := c.md5Hasher() md5Hash.Write(buf[:length]) md5Base64 = base64.StdEncoding.EncodeToString(md5Hash.Sum(nil)) md5Hash.Close() } defer wg.Done() p := uploadPartParams{ bucketName: bucketName, objectName: objectName, uploadID: uploadID, reader: bytes.NewReader(buf[:length]), partNumber: partNumber, md5Base64: md5Base64, size: int64(length), sse: opts.ServerSideEncryption, streamSha256: !opts.DisableContentSha256, customHeader: customHeader, } objPart, uerr := c.uploadPart(ctx, p) if uerr != nil { errCh <- uerr return } // Save successfully uploaded part metadata. mu.Lock() partsInfo[partNumber] = objPart mu.Unlock() // Send buffer back so it can be reused. bufs <- buf }(partNumber) // Save successfully uploaded size. totalUploadedSize += int64(length) } wg.Wait() // Collect any error select { case err = <-errCh: return UploadInfo{}, err default: } // Complete multipart upload. var complMultipartUpload completeMultipartUpload // Loop over total uploaded parts to save them in // Parts array before completing the multipart request. allParts := make([]ObjectPart, 0, len(partsInfo)) for i := 1; i < partNumber; i++ { part, ok := partsInfo[i] if !ok { return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } allParts = append(allParts, part) complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, PartNumber: part.PartNumber, ChecksumCRC32: part.ChecksumCRC32, ChecksumCRC32C: part.ChecksumCRC32C, ChecksumSHA1: part.ChecksumSHA1, ChecksumSHA256: part.ChecksumSHA256, ChecksumCRC64NVME: part.ChecksumCRC64NVME, }) } // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) opts = PutObjectOptions{ ServerSideEncryption: opts.ServerSideEncryption, AutoChecksum: opts.AutoChecksum, } applyAutoChecksum(&opts, allParts) uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts) if err != nil { return UploadInfo{}, err } uploadInfo.Size = totalUploadedSize return uploadInfo, nil } // putObject special function used Google Cloud Storage. This special function // is used for Google Cloud Storage since Google's multipart API is not S3 compatible. func (c *Client) putObject(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return UploadInfo{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return UploadInfo{}, err } // Size -1 is only supported on Google Cloud Storage, we error // out in all other situations. if size < 0 && !s3utils.IsGoogleEndpoint(*c.endpointURL) { return UploadInfo{}, errEntityTooSmall(size, bucketName, objectName) } if opts.SendContentMd5 && s3utils.IsGoogleEndpoint(*c.endpointURL) && size < 0 { return UploadInfo{}, errInvalidArgument("MD5Sum cannot be calculated with size '-1'") } var readSeeker io.Seeker if size > 0 { if isReadAt(reader) && !isObject(reader) { seeker, ok := reader.(io.Seeker) if ok { offset, err := seeker.Seek(0, io.SeekCurrent) if err != nil { return UploadInfo{}, errInvalidArgument(err.Error()) } reader = io.NewSectionReader(reader.(io.ReaderAt), offset, size) readSeeker = reader.(io.Seeker) } } } var md5Base64 string if opts.SendContentMd5 { // Calculate md5sum. hash := c.md5Hasher() if readSeeker != nil { if _, err := io.Copy(hash, reader); err != nil { return UploadInfo{}, err } // Seek back to beginning of io.NewSectionReader's offset. _, err = readSeeker.Seek(0, io.SeekStart) if err != nil { return UploadInfo{}, errInvalidArgument(err.Error()) } } else { // Create a buffer. buf := make([]byte, size) length, err := readFull(reader, buf) if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF { return UploadInfo{}, err } hash.Write(buf[:length]) reader = bytes.NewReader(buf[:length]) } md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil)) hash.Close() } // Update progress reader appropriately to the latest offset as we // read from the source. progressReader := newHook(reader, opts.Progress) // This function does not calculate sha256 and md5sum for payload. // Execute put object. return c.putObjectDo(ctx, bucketName, objectName, progressReader, md5Base64, "", size, opts) } // putObjectDo - executes the put object http operation. // NOTE: You must have WRITE permissions on a bucket to add an object to it. func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string, reader io.Reader, md5Base64, sha256Hex string, size int64, opts PutObjectOptions) (UploadInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return UploadInfo{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return UploadInfo{}, err } // Set headers. customHeader := opts.Header() // Populate request metadata. reqMetadata := requestMetadata{ bucketName: bucketName, objectName: objectName, customHeader: customHeader, contentBody: reader, contentLength: size, contentMD5Base64: md5Base64, contentSHA256Hex: sha256Hex, streamSha256: !opts.DisableContentSha256, } // Add CRC when client supports it, MD5 is not set, not Google and we don't add SHA256 to chunks. addCrc := c.trailingHeaderSupport && md5Base64 == "" && !s3utils.IsGoogleEndpoint(*c.endpointURL) && (opts.DisableContentSha256 || c.secure) if opts.Checksum.IsSet() { reqMetadata.addCrc = &opts.Checksum } else if addCrc { // If user has added checksums, don't add them ourselves. for k := range opts.UserMetadata { if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") { addCrc = false } } if addCrc { opts.AutoChecksum.SetDefault(ChecksumFullObjectCRC32C) reqMetadata.addCrc = &opts.AutoChecksum } } if opts.Internal.SourceVersionID != "" { if opts.Internal.SourceVersionID != nullVersionID { if _, err := uuid.Parse(opts.Internal.SourceVersionID); err != nil { return UploadInfo{}, errInvalidArgument(err.Error()) } } urlValues := make(url.Values) urlValues.Set("versionId", opts.Internal.SourceVersionID) reqMetadata.queryValues = urlValues } // Execute PUT an objectName. resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata) defer closeResponse(resp) if err != nil { return UploadInfo{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { return UploadInfo{}, httpRespToErrorResponse(resp, bucketName, objectName) } } // extract lifecycle expiry date and rule ID expTime, ruleID := amzExpirationToExpiryDateRuleID(resp.Header.Get(amzExpiration)) h := resp.Header return UploadInfo{ Bucket: bucketName, Key: objectName, ETag: trimEtag(h.Get("ETag")), VersionID: h.Get(amzVersionID), Size: size, Expiration: expTime, ExpirationRuleID: ruleID, // Checksum values ChecksumCRC32: h.Get(ChecksumCRC32.Key()), ChecksumCRC32C: h.Get(ChecksumCRC32C.Key()), ChecksumSHA1: h.Get(ChecksumSHA1.Key()), ChecksumSHA256: h.Get(ChecksumSHA256.Key()), ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()), ChecksumMode: h.Get(ChecksumFullObjectMode.Key()), }, nil } minio-go-7.0.97/api-put-object.go000066400000000000000000000405111510244170000165010ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/base64" "errors" "fmt" "io" "net/http" "sort" "strings" "time" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/s3utils" "github.com/minio/minio-go/v7/pkg/tags" "golang.org/x/net/http/httpguts" ) // ReplicationStatus represents replication status of object type ReplicationStatus string const ( // ReplicationStatusPending indicates replication is pending ReplicationStatusPending ReplicationStatus = "PENDING" // ReplicationStatusComplete indicates replication completed ok ReplicationStatusComplete ReplicationStatus = "COMPLETED" // ReplicationStatusFailed indicates replication failed ReplicationStatusFailed ReplicationStatus = "FAILED" // ReplicationStatusReplica indicates object is a replica of a source ReplicationStatusReplica ReplicationStatus = "REPLICA" // ReplicationStatusReplicaEdge indicates object is a replica of a edge source ReplicationStatusReplicaEdge ReplicationStatus = "REPLICA-EDGE" ) // Empty returns true if no replication status set. func (r ReplicationStatus) Empty() bool { return r == "" } // AdvancedPutOptions for internal use - to be utilized by replication, ILM transition // implementation on MinIO server type AdvancedPutOptions struct { SourceVersionID string SourceETag string ReplicationStatus ReplicationStatus SourceMTime time.Time ReplicationRequest bool RetentionTimestamp time.Time TaggingTimestamp time.Time LegalholdTimestamp time.Time ReplicationValidityCheck bool } // PutObjectOptions represents options specified by user for PutObject call type PutObjectOptions struct { UserMetadata map[string]string UserTags map[string]string Progress io.Reader ContentType string ContentEncoding string ContentDisposition string ContentLanguage string CacheControl string Expires time.Time Mode RetentionMode RetainUntilDate time.Time ServerSideEncryption encrypt.ServerSide NumThreads uint StorageClass string WebsiteRedirectLocation string PartSize uint64 LegalHold LegalHoldStatus SendContentMd5 bool DisableContentSha256 bool DisableMultipart bool // AutoChecksum is the type of checksum that will be added if no other checksum is added, // like MD5 or SHA256 streaming checksum, and it is feasible for the upload type. // If none is specified CRC32C is used, since it is generally the fastest. AutoChecksum ChecksumType // Checksum will force a checksum of the specific type. // This requires that the client was created with "TrailingHeaders:true" option, // and that the destination server supports it. // Unavailable with V2 signatures & Google endpoints. // This will disable content MD5 checksums if set. Checksum ChecksumType // ConcurrentStreamParts will create NumThreads buffers of PartSize bytes, // fill them serially and upload them in parallel. // This can be used for faster uploads on non-seekable or slow-to-seek input. ConcurrentStreamParts bool Internal AdvancedPutOptions customHeaders http.Header } // SetMatchETag if etag matches while PUT MinIO returns an error // this is a MinIO specific extension to support optimistic locking // semantics. func (opts *PutObjectOptions) SetMatchETag(etag string) { if opts.customHeaders == nil { opts.customHeaders = http.Header{} } if etag == "*" { opts.customHeaders.Set("If-Match", "*") } else { opts.customHeaders.Set("If-Match", "\""+etag+"\"") } } // SetMatchETagExcept if etag does not match while PUT MinIO returns an // error this is a MinIO specific extension to support optimistic locking // semantics. func (opts *PutObjectOptions) SetMatchETagExcept(etag string) { if opts.customHeaders == nil { opts.customHeaders = http.Header{} } if etag == "*" { opts.customHeaders.Set("If-None-Match", "*") } else { opts.customHeaders.Set("If-None-Match", "\""+etag+"\"") } } // getNumThreads - gets the number of threads to be used in the multipart // put object operation func (opts PutObjectOptions) getNumThreads() (numThreads int) { if opts.NumThreads > 0 { numThreads = int(opts.NumThreads) } else { numThreads = totalWorkers } return numThreads } // Header - constructs the headers from metadata entered by user in // PutObjectOptions struct func (opts PutObjectOptions) Header() (header http.Header) { header = make(http.Header) contentType := opts.ContentType if contentType == "" { contentType = "application/octet-stream" } header.Set("Content-Type", contentType) if opts.ContentEncoding != "" { header.Set("Content-Encoding", opts.ContentEncoding) } if opts.ContentDisposition != "" { header.Set("Content-Disposition", opts.ContentDisposition) } if opts.ContentLanguage != "" { header.Set("Content-Language", opts.ContentLanguage) } if opts.CacheControl != "" { header.Set("Cache-Control", opts.CacheControl) } if !opts.Expires.IsZero() { header.Set("Expires", opts.Expires.UTC().Format(http.TimeFormat)) } if opts.Mode != "" { header.Set(amzLockMode, opts.Mode.String()) } if !opts.RetainUntilDate.IsZero() { header.Set("X-Amz-Object-Lock-Retain-Until-Date", opts.RetainUntilDate.Format(time.RFC3339)) } if opts.LegalHold != "" { header.Set(amzLegalHoldHeader, opts.LegalHold.String()) } if opts.ServerSideEncryption != nil { opts.ServerSideEncryption.Marshal(header) } if opts.StorageClass != "" { header.Set(amzStorageClass, opts.StorageClass) } if opts.WebsiteRedirectLocation != "" { header.Set(amzWebsiteRedirectLocation, opts.WebsiteRedirectLocation) } if !opts.Internal.ReplicationStatus.Empty() { header.Set(amzBucketReplicationStatus, string(opts.Internal.ReplicationStatus)) } if !opts.Internal.SourceMTime.IsZero() { header.Set(minIOBucketSourceMTime, opts.Internal.SourceMTime.Format(time.RFC3339Nano)) } if opts.Internal.SourceETag != "" { header.Set(minIOBucketSourceETag, opts.Internal.SourceETag) } if opts.Internal.ReplicationRequest { header.Set(minIOBucketReplicationRequest, "true") } if opts.Internal.ReplicationValidityCheck { header.Set(minIOBucketReplicationCheck, "true") } if !opts.Internal.LegalholdTimestamp.IsZero() { header.Set(minIOBucketReplicationObjectLegalHoldTimestamp, opts.Internal.LegalholdTimestamp.Format(time.RFC3339Nano)) } if !opts.Internal.RetentionTimestamp.IsZero() { header.Set(minIOBucketReplicationObjectRetentionTimestamp, opts.Internal.RetentionTimestamp.Format(time.RFC3339Nano)) } if !opts.Internal.TaggingTimestamp.IsZero() { header.Set(minIOBucketReplicationTaggingTimestamp, opts.Internal.TaggingTimestamp.Format(time.RFC3339Nano)) } if len(opts.UserTags) != 0 { if tags, _ := tags.NewTags(opts.UserTags, true); tags != nil { header.Set(amzTaggingHeader, tags.String()) } } for k, v := range opts.UserMetadata { if isAmzHeader(k) || isStandardHeader(k) || isStorageClassHeader(k) || isMinioHeader(k) { header.Set(k, v) } else { header.Set("x-amz-meta-"+k, v) } } // set any other additional custom headers. for k, v := range opts.customHeaders { header[k] = v } return header } // validate() checks if the UserMetadata map has standard headers or and raises an error if so. func (opts PutObjectOptions) validate(c *Client) (err error) { for k, v := range opts.UserMetadata { if !httpguts.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) || isMinioHeader(k) { return errInvalidArgument(k + " unsupported user defined metadata name") } if !httpguts.ValidHeaderFieldValue(v) { return errInvalidArgument(v + " unsupported user defined metadata value") } } if opts.Mode != "" && !opts.Mode.IsValid() { return errInvalidArgument(opts.Mode.String() + " unsupported retention mode") } if opts.LegalHold != "" && !opts.LegalHold.IsValid() { return errInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status") } checkCrc := false for k := range opts.UserMetadata { if strings.HasPrefix(k, "x-amz-checksum-") { checkCrc = true break } } if opts.Checksum.IsSet() || checkCrc { switch { case !c.trailingHeaderSupport: return errInvalidArgument("Checksum requires Client with TrailingHeaders enabled") case c.overrideSignerType.IsV2(): return errInvalidArgument("Checksum cannot be used with v2 signatures") case s3utils.IsGoogleEndpoint(*c.endpointURL): return errInvalidArgument("Checksum cannot be used with GCS endpoints") } } return nil } // completedParts is a collection of parts sortable by their part numbers. // used for sorting the uploaded parts before completing the multipart request. type completedParts []CompletePart func (a completedParts) Len() int { return len(a) } func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber } // PutObject creates an object in a bucket. // // You must have WRITE permissions on a bucket to create an object. // // - For size smaller than 16MiB PutObject automatically does a // single atomic PUT operation. // // - For size larger than 16MiB PutObject automatically does a // multipart upload operation. // // - For size input as -1 PutObject does a multipart Put operation // until input stream reaches EOF. Maximum object size that can // be uploaded through this operation will be 5TiB. // // WARNING: Passing down '-1' will use memory and these cannot // be reused for best outcomes for PutObject(), pass the size always. // // NOTE: Upon errors during upload multipart operation is entirely aborted. func (c *Client) PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, size int64, opts PutObjectOptions, ) (info UploadInfo, err error) { if size < 0 && opts.DisableMultipart { return UploadInfo{}, errors.New("object size must be provided with disable multipart upload") } err = opts.validate(c) if err != nil { return UploadInfo{}, err } // Check for largest object size allowed. if size > int64(maxMultipartPutObjectSize) { return UploadInfo{}, errEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName) } if opts.Checksum.IsSet() { opts.AutoChecksum = opts.Checksum opts.SendContentMd5 = false } if c.trailingHeaderSupport { opts.AutoChecksum.SetDefault(ChecksumCRC32C) addAutoChecksumHeaders(&opts) } // NOTE: Streaming signature is not supported by GCS. if s3utils.IsGoogleEndpoint(*c.endpointURL) { return c.putObject(ctx, bucketName, objectName, reader, size, opts) } partSize := opts.PartSize if opts.PartSize == 0 { partSize = minPartSize } if c.overrideSignerType.IsV2() { if size >= 0 && size < int64(partSize) || opts.DisableMultipart { return c.putObject(ctx, bucketName, objectName, reader, size, opts) } return c.putObjectMultipart(ctx, bucketName, objectName, reader, size, opts) } if size < 0 { if opts.DisableMultipart { return UploadInfo{}, errors.New("no length provided and multipart disabled") } if opts.ConcurrentStreamParts && opts.NumThreads > 1 { return c.putObjectMultipartStreamParallel(ctx, bucketName, objectName, reader, opts) } return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts) } if size <= int64(partSize) || opts.DisableMultipart { return c.putObject(ctx, bucketName, objectName, reader, size, opts) } return c.putObjectMultipartStream(ctx, bucketName, objectName, reader, size, opts) } func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName, objectName string, reader io.Reader, opts PutObjectOptions) (info UploadInfo, err error) { // Input validation. if err = s3utils.CheckValidBucketName(bucketName); err != nil { return UploadInfo{}, err } if err = s3utils.CheckValidObjectName(objectName); err != nil { return UploadInfo{}, err } // Total data read and written to server. should be equal to // 'size' at the end of the call. var totalUploadedSize int64 // Complete multipart upload. var complMultipartUpload completeMultipartUpload // Calculate the optimal parts info for a given size. totalPartsCount, partSize, _, err := OptimalPartInfo(-1, opts.PartSize) if err != nil { return UploadInfo{}, err } // Initiate a new multipart upload. uploadID, err := c.newUploadID(ctx, bucketName, objectName, opts) if err != nil { return UploadInfo{}, err } defer func() { if err != nil { c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) } }() // Part number always starts with '1'. partNumber := 1 // Initialize parts uploaded map. partsInfo := make(map[int]ObjectPart) // Create a buffer. buf := make([]byte, partSize) // Create checksums // CRC32C is ~50% faster on AMD64 @ 30GB/s customHeader := make(http.Header) crc := opts.AutoChecksum.Hasher() for partNumber <= totalPartsCount { length, rerr := readFull(reader, buf) if rerr == io.EOF && partNumber > 1 { break } if rerr != nil && rerr != io.ErrUnexpectedEOF && rerr != io.EOF { return UploadInfo{}, rerr } var md5Base64 string if opts.SendContentMd5 { // Calculate md5sum. hash := c.md5Hasher() hash.Write(buf[:length]) md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil)) hash.Close() } if opts.AutoChecksum.IsSet() { crc.Reset() crc.Write(buf[:length]) cSum := crc.Sum(nil) customHeader.Set(opts.AutoChecksum.Key(), base64.StdEncoding.EncodeToString(cSum)) customHeader.Set(amzChecksumAlgo, opts.AutoChecksum.String()) if opts.AutoChecksum.FullObjectRequested() { customHeader.Set(amzChecksumMode, ChecksumFullObjectMode.String()) } } // Update progress reader appropriately to the latest offset // as we read from the source. rd := newHook(bytes.NewReader(buf[:length]), opts.Progress) // Proceed to upload the part. p := uploadPartParams{bucketName: bucketName, objectName: objectName, uploadID: uploadID, reader: rd, partNumber: partNumber, md5Base64: md5Base64, size: int64(length), sse: opts.ServerSideEncryption, streamSha256: !opts.DisableContentSha256, customHeader: customHeader} objPart, uerr := c.uploadPart(ctx, p) if uerr != nil { return UploadInfo{}, uerr } // Save successfully uploaded part metadata. partsInfo[partNumber] = objPart // Save successfully uploaded size. totalUploadedSize += int64(length) // Increment part number. partNumber++ // For unknown size, Read EOF we break away. // We do not have to upload till totalPartsCount. if rerr == io.EOF { break } } // Loop over total uploaded parts to save them in // Parts array before completing the multipart request. allParts := make([]ObjectPart, 0, len(partsInfo)) for i := 1; i < partNumber; i++ { part, ok := partsInfo[i] if !ok { return UploadInfo{}, errInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } allParts = append(allParts, part) complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, PartNumber: part.PartNumber, ChecksumCRC32: part.ChecksumCRC32, ChecksumCRC32C: part.ChecksumCRC32C, ChecksumSHA1: part.ChecksumSHA1, ChecksumSHA256: part.ChecksumSHA256, ChecksumCRC64NVME: part.ChecksumCRC64NVME, }) } // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) opts = PutObjectOptions{ ServerSideEncryption: opts.ServerSideEncryption, AutoChecksum: opts.AutoChecksum, } applyAutoChecksum(&opts, allParts) uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, opts) if err != nil { return UploadInfo{}, err } uploadInfo.Size = totalUploadedSize return uploadInfo, nil } minio-go-7.0.97/api-put-object_test.go000066400000000000000000000117441510244170000175460ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "encoding/base64" "net/http" "reflect" "testing" "github.com/minio/minio-go/v7/pkg/encrypt" ) func TestPutObjectOptionsValidate(t *testing.T) { testCases := []struct { name, value string shouldPass bool }{ // Invalid cases. {"X-Amz-Matdesc", "blah", false}, {"x-amz-meta-X-Amz-Iv", "blah", false}, {"x-amz-meta-X-Amz-Key", "blah", false}, {"x-amz-meta-X-Amz-Matdesc", "blah", false}, {"It has spaces", "v", false}, {"It,has@illegal=characters", "v", false}, {"X-Amz-Iv", "blah", false}, {"X-Amz-Key", "blah", false}, {"X-Amz-Key-prefixed-header", "blah", false}, {"Content-Type", "custom/content-type", false}, {"content-type", "custom/content-type", false}, {"Content-Encoding", "gzip", false}, {"Cache-Control", "blah", false}, {"Content-Disposition", "something", false}, {"Content-Language", "somelanguage", false}, // Valid metadata names. {"my-custom-header", "blah", true}, {"custom-X-Amz-Key-middle", "blah", true}, {"my-custom-header-X-Amz-Key", "blah", true}, {"blah-X-Amz-Matdesc", "blah", true}, {"X-Amz-MatDesc-suffix", "blah", true}, {"It-Is-Fine", "v", true}, {"Numbers-098987987-Should-Work", "v", true}, {"Crazy-!#$%&'*+-.^_`|~-Should-193832-Be-Fine", "v", true}, } for i, testCase := range testCases { err := PutObjectOptions{UserMetadata: map[string]string{ testCase.name: testCase.value, }}.validate(nil) if testCase.shouldPass && err != nil { t.Errorf("Test %d - output did not match with reference results, %s", i+1, err) } } } type InterceptRouteTripper struct { request *http.Request } func (i *InterceptRouteTripper) RoundTrip(request *http.Request) (*http.Response, error) { i.request = request return &http.Response{StatusCode: 200}, nil } func Test_SSEHeaders(t *testing.T) { rt := &InterceptRouteTripper{} c, err := New("s3.amazonaws.com", &Options{ Transport: rt, }) if err != nil { t.Error(err) } testCases := map[string]struct { sse func() encrypt.ServerSide initiateMultipartUploadHeaders http.Header headerNotAllowedAfterInit []string }{ "noEncryption": { sse: func() encrypt.ServerSide { return nil }, initiateMultipartUploadHeaders: http.Header{}, }, "sse": { sse: func() encrypt.ServerSide { s, err := encrypt.NewSSEKMS("keyId", nil) if err != nil { t.Error(err) } return s }, initiateMultipartUploadHeaders: http.Header{ encrypt.SseGenericHeader: []string{"aws:kms"}, encrypt.SseKmsKeyID: []string{"keyId"}, }, headerNotAllowedAfterInit: []string{encrypt.SseGenericHeader, encrypt.SseKmsKeyID, encrypt.SseEncryptionContext}, }, "sse with context": { sse: func() encrypt.ServerSide { s, err := encrypt.NewSSEKMS("keyId", "context") if err != nil { t.Error(err) } return s }, initiateMultipartUploadHeaders: http.Header{ encrypt.SseGenericHeader: []string{"aws:kms"}, encrypt.SseKmsKeyID: []string{"keyId"}, encrypt.SseEncryptionContext: []string{base64.StdEncoding.EncodeToString([]byte("\"context\""))}, }, headerNotAllowedAfterInit: []string{encrypt.SseGenericHeader, encrypt.SseKmsKeyID, encrypt.SseEncryptionContext}, }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { opts := PutObjectOptions{ ServerSideEncryption: tc.sse(), } c.bucketLocCache.Set("test", "region") c.initiateMultipartUpload(context.Background(), "test", "test", opts) for s, vls := range tc.initiateMultipartUploadHeaders { if !reflect.DeepEqual(rt.request.Header[s], vls) { t.Errorf("Header %v are not equal, want: %v got %v", s, vls, rt.request.Header[s]) } } _, err := c.uploadPart(context.Background(), uploadPartParams{ bucketName: "test", objectName: "test", partNumber: 1, uploadID: "upId", sse: opts.ServerSideEncryption, }) if err != nil { t.Error(err) } for _, k := range tc.headerNotAllowedAfterInit { if rt.request.Header.Get(k) != "" { t.Errorf("header %v should not be set", k) } } c.completeMultipartUpload(context.Background(), "test", "test", "upId", completeMultipartUpload{}, opts) for _, k := range tc.headerNotAllowedAfterInit { if rt.request.Header.Get(k) != "" { t.Errorf("header %v should not be set", k) } } }) } } minio-go-7.0.97/api-putobject-snowball.go000066400000000000000000000140031510244170000202400ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "archive/tar" "bufio" "bytes" "context" "fmt" "io" "net/http" "os" "strings" "sync" "time" "github.com/klauspost/compress/s2" ) // SnowballOptions contains options for PutObjectsSnowball calls. type SnowballOptions struct { // Opts is options applied to all objects. Opts PutObjectOptions // Processing options: // InMemory specifies that all objects should be collected in memory // before they are uploaded. // If false a temporary file will be created. InMemory bool // Compress enabled content compression before upload. // Compression will typically reduce memory and network usage, // Compression can safely be enabled with MinIO hosts. Compress bool // SkipErrs if enabled will skip any errors while reading the // object content while creating the snowball archive SkipErrs bool } // SnowballObject contains information about a single object to be added to the snowball. type SnowballObject struct { // Key is the destination key, including prefix. Key string // Size is the content size of this object. Size int64 // Modtime to apply to the object. // If Modtime is the zero value current time will be used. ModTime time.Time // Content of the object. // Exactly 'Size' number of bytes must be provided. Content io.Reader // VersionID of the object; if empty, a new versionID will be generated VersionID string // Headers contains more options for this object upload, the same as you // would include in a regular PutObject operation, such as user metadata // and content-disposition, expires, .. Headers http.Header // Close will be called when an object has finished processing. // Note that if PutObjectsSnowball returns because of an error, // objects not consumed from the input will NOT have been closed. // Leave as nil for no callback. Close func() } type nopReadSeekCloser struct { io.ReadSeeker } func (n nopReadSeekCloser) Close() error { return nil } // This is available as io.ReadSeekCloser from go1.16 type readSeekCloser interface { io.Reader io.Closer io.Seeker } // PutObjectsSnowball will put multiple objects with a single put call. // A (compressed) TAR file will be created which will contain multiple objects. // The key for each object will be used for the destination in the specified bucket. // Total size should be < 5TB. // This function blocks until 'objs' is closed and the content has been uploaded. func (c *Client) PutObjectsSnowball(ctx context.Context, bucketName string, opts SnowballOptions, objs <-chan SnowballObject) (err error) { err = opts.Opts.validate(c) if err != nil { return err } var tmpWriter io.Writer var getTmpReader func() (rc readSeekCloser, sz int64, err error) if opts.InMemory { b := bytes.NewBuffer(nil) tmpWriter = b getTmpReader = func() (readSeekCloser, int64, error) { return nopReadSeekCloser{bytes.NewReader(b.Bytes())}, int64(b.Len()), nil } } else { f, err := os.CreateTemp("", "s3-putsnowballobjects-*") if err != nil { return err } name := f.Name() tmpWriter = f var once sync.Once defer once.Do(func() { f.Close() }) defer os.Remove(name) getTmpReader = func() (readSeekCloser, int64, error) { once.Do(func() { f.Close() }) f, err := os.Open(name) if err != nil { return nil, 0, err } st, err := f.Stat() if err != nil { return nil, 0, err } return f, st.Size(), nil } } flush := func() error { return nil } if !opts.Compress { if !opts.InMemory { // Insert buffer for writes. buf := bufio.NewWriterSize(tmpWriter, 1<<20) flush = buf.Flush tmpWriter = buf } } else { s2c := s2.NewWriter(tmpWriter, s2.WriterBetterCompression()) flush = s2c.Close defer s2c.Close() tmpWriter = s2c } t := tar.NewWriter(tmpWriter) objectLoop: for { select { case <-ctx.Done(): return ctx.Err() case obj, ok := <-objs: if !ok { break objectLoop } closeObj := func() {} if obj.Close != nil { closeObj = obj.Close } // Trim accidental slash prefix. obj.Key = strings.TrimPrefix(obj.Key, "/") header := tar.Header{ Typeflag: tar.TypeReg, Name: obj.Key, Size: obj.Size, ModTime: obj.ModTime, Format: tar.FormatPAX, } if header.ModTime.IsZero() { header.ModTime = time.Now().UTC() } header.PAXRecords = make(map[string]string) if obj.VersionID != "" { header.PAXRecords["minio.versionId"] = obj.VersionID } for k, vals := range obj.Headers { header.PAXRecords["minio.metadata."+k] = strings.Join(vals, ",") } if err := t.WriteHeader(&header); err != nil { closeObj() return err } n, err := io.Copy(t, obj.Content) if err != nil { closeObj() if opts.SkipErrs { continue } return err } if n != obj.Size { closeObj() if opts.SkipErrs { continue } return io.ErrUnexpectedEOF } closeObj() } } // Flush tar err = t.Flush() if err != nil { return err } // Flush compression err = flush() if err != nil { return err } if opts.Opts.UserMetadata == nil { opts.Opts.UserMetadata = map[string]string{} } opts.Opts.UserMetadata["X-Amz-Meta-Snowball-Auto-Extract"] = "true" opts.Opts.DisableMultipart = true rc, sz, err := getTmpReader() if err != nil { return err } defer rc.Close() rand := c.random.Uint64() _, err = c.PutObject(ctx, bucketName, fmt.Sprintf("snowball-upload-%x.tar", rand), rc, sz, opts.Opts) return err } minio-go-7.0.97/api-remove.go000066400000000000000000000510251510244170000157240ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/xml" "io" "iter" "net/http" "net/url" "time" "github.com/minio/minio-go/v7/pkg/s3utils" ) //revive:disable // Deprecated: BucketOptions will be renamed to RemoveBucketOptions in future versions. type BucketOptions = RemoveBucketOptions //revive:enable // RemoveBucketOptions special headers to purge buckets, only // useful when endpoint is MinIO type RemoveBucketOptions struct { ForceDelete bool } // RemoveBucketWithOptions deletes the bucket name. // // All objects (including all object versions and delete markers) // in the bucket will be deleted forcibly if bucket options set // ForceDelete to 'true'. func (c *Client) RemoveBucketWithOptions(ctx context.Context, bucketName string, opts RemoveBucketOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } // Build headers. headers := make(http.Header) if opts.ForceDelete { headers.Set(minIOForceDelete, "true") } // Execute DELETE on bucket. resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, contentSHA256Hex: emptySHA256Hex, customHeader: headers, }) defer closeResponse(resp) if err != nil { return err } if resp != nil { if resp.StatusCode != http.StatusNoContent { return httpRespToErrorResponse(resp, bucketName, "") } } // Remove the location from cache on a successful delete. c.bucketLocCache.Delete(bucketName) return nil } // RemoveBucket deletes the bucket name. // // All objects (including all object versions and delete markers). // in the bucket must be deleted before successfully attempting this request. func (c *Client) RemoveBucket(ctx context.Context, bucketName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } // Execute DELETE on bucket. resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return err } if resp != nil { if resp.StatusCode != http.StatusNoContent { return httpRespToErrorResponse(resp, bucketName, "") } } // Remove the location from cache on a successful delete. c.bucketLocCache.Delete(bucketName) return nil } // AdvancedRemoveOptions intended for internal use by replication type AdvancedRemoveOptions struct { ReplicationDeleteMarker bool ReplicationStatus ReplicationStatus ReplicationMTime time.Time ReplicationRequest bool ReplicationValidityCheck bool // check permissions } // RemoveObjectOptions represents options specified by user for RemoveObject call type RemoveObjectOptions struct { ForceDelete bool GovernanceBypass bool VersionID string Internal AdvancedRemoveOptions } // RemoveObject removes an object from a bucket. func (c *Client) RemoveObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return err } res := c.removeObject(ctx, bucketName, objectName, opts) return res.Err } func (c *Client) removeObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) RemoveObjectResult { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) if opts.VersionID != "" { urlValues.Set("versionId", opts.VersionID) } // Build headers. headers := make(http.Header) if opts.GovernanceBypass { // Set the bypass goverenance retention header headers.Set(amzBypassGovernance, "true") } if opts.Internal.ReplicationDeleteMarker { headers.Set(minIOBucketReplicationDeleteMarker, "true") } if !opts.Internal.ReplicationMTime.IsZero() { headers.Set(minIOBucketSourceMTime, opts.Internal.ReplicationMTime.Format(time.RFC3339Nano)) } if !opts.Internal.ReplicationStatus.Empty() { headers.Set(amzBucketReplicationStatus, string(opts.Internal.ReplicationStatus)) } if opts.Internal.ReplicationRequest { headers.Set(minIOBucketReplicationRequest, "true") } if opts.Internal.ReplicationValidityCheck { headers.Set(minIOBucketReplicationCheck, "true") } if opts.ForceDelete { headers.Set(minIOForceDelete, "true") } // Execute DELETE on objectName. resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, objectName: objectName, contentSHA256Hex: emptySHA256Hex, queryValues: urlValues, customHeader: headers, }) defer closeResponse(resp) if err != nil { return RemoveObjectResult{Err: err} } if resp != nil { // if some unexpected error happened and max retry is reached, we want to let client know if resp.StatusCode != http.StatusNoContent { err := httpRespToErrorResponse(resp, bucketName, objectName) return RemoveObjectResult{Err: err} } } // DeleteObject always responds with http '204' even for // objects which do not exist. So no need to handle them // specifically. return RemoveObjectResult{ ObjectName: objectName, ObjectVersionID: opts.VersionID, DeleteMarker: resp.Header.Get("x-amz-delete-marker") == "true", DeleteMarkerVersionID: resp.Header.Get("x-amz-version-id"), } } // RemoveObjectError - container of Multi Delete S3 API error type RemoveObjectError struct { ObjectName string VersionID string Err error } func (err *RemoveObjectError) Error() string { // This should never happen as we will have a non-nil error with no underlying error. if err.Err == nil { return "unexpected remove object error result" } return err.Err.Error() } // RemoveObjectResult - container of Multi Delete S3 API result type RemoveObjectResult struct { ObjectName string ObjectVersionID string DeleteMarker bool DeleteMarkerVersionID string Err error } // generateRemoveMultiObjects - generate the XML request for remove multi objects request func generateRemoveMultiObjectsRequest(objects []ObjectInfo) []byte { delObjects := []deleteObject{} for _, obj := range objects { delObjects = append(delObjects, deleteObject{ Key: obj.Key, VersionID: obj.VersionID, }) } xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: delObjects, Quiet: false}) return xmlBytes } // processRemoveMultiObjectsResponse - parse the remove multi objects web service // and return the success/failure result status for each object func processRemoveMultiObjectsResponse(body io.Reader, resultCh chan<- RemoveObjectResult) { // Parse multi delete XML response rmResult := &deleteMultiObjectsResult{} err := xmlDecoder(body, rmResult) if err != nil { resultCh <- RemoveObjectResult{ObjectName: "", Err: err} return } // Fill deletion that returned success for _, obj := range rmResult.DeletedObjects { resultCh <- RemoveObjectResult{ ObjectName: obj.Key, // Only filled with versioned buckets ObjectVersionID: obj.VersionID, DeleteMarker: obj.DeleteMarker, DeleteMarkerVersionID: obj.DeleteMarkerVersionID, } } // Fill deletion that returned an error. for _, obj := range rmResult.UnDeletedObjects { // Version does not exist is not an error ignore and continue. switch obj.Code { case InvalidArgument, NoSuchVersion: continue } resultCh <- RemoveObjectResult{ ObjectName: obj.Key, ObjectVersionID: obj.VersionID, Err: ErrorResponse{ Code: obj.Code, Message: obj.Message, }, } } } // RemoveObjectsOptions represents options specified by user for RemoveObjects call type RemoveObjectsOptions struct { GovernanceBypass bool } // RemoveObjects removes multiple objects from a bucket while // it is possible to specify objects versions which are received from // objectsCh. Remove failures are sent back via error channel. func (c *Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, opts RemoveObjectsOptions) <-chan RemoveObjectError { errorCh := make(chan RemoveObjectError, 1) // Validate if bucket name is valid. if err := s3utils.CheckValidBucketName(bucketName); err != nil { defer close(errorCh) errorCh <- RemoveObjectError{ Err: err, } return errorCh } // Validate objects channel to be properly allocated. if objectsCh == nil { defer close(errorCh) errorCh <- RemoveObjectError{ Err: errInvalidArgument("Objects channel cannot be nil"), } return errorCh } resultCh := make(chan RemoveObjectResult, 1) go c.removeObjects(ctx, bucketName, objectsCh, resultCh, opts) go func() { defer close(errorCh) for res := range resultCh { // Send only errors to the error channel if res.Err == nil { continue } errorCh <- RemoveObjectError{ ObjectName: res.ObjectName, VersionID: res.ObjectVersionID, Err: res.Err, } } }() return errorCh } // RemoveObjectsWithIter bulk deletes multiple objects from a bucket. // Objects (with optional versions) to be removed must be provided with // an iterator. Objects are removed asynchronously and results must be // consumed. If the returned result iterator is stopped, the context is // canceled, or a remote call failed, the provided iterator will no // longer accept more objects. func (c *Client) RemoveObjectsWithIter(ctx context.Context, bucketName string, objectsIter iter.Seq[ObjectInfo], opts RemoveObjectsOptions) (iter.Seq[RemoveObjectResult], error) { // Validate if bucket name is valid. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err } // Validate objects channel to be properly allocated. if objectsIter == nil { return nil, errInvalidArgument("Objects iter can never by nil") } return func(yield func(RemoveObjectResult) bool) { select { case <-ctx.Done(): return default: } c.removeObjectsIter(ctx, bucketName, objectsIter, yield, opts) }, nil } // RemoveObjectsWithResult removes multiple objects from a bucket while // it is possible to specify objects versions which are received from // objectsCh. Remove results, successes and failures are sent back via // RemoveObjectResult channel func (c *Client) RemoveObjectsWithResult(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, opts RemoveObjectsOptions) <-chan RemoveObjectResult { resultCh := make(chan RemoveObjectResult, 1) // Validate if bucket name is valid. if err := s3utils.CheckValidBucketName(bucketName); err != nil { defer close(resultCh) resultCh <- RemoveObjectResult{ Err: err, } return resultCh } // Validate objects channel to be properly allocated. if objectsCh == nil { defer close(resultCh) resultCh <- RemoveObjectResult{ Err: errInvalidArgument("Objects channel cannot be nil"), } return resultCh } go c.removeObjects(ctx, bucketName, objectsCh, resultCh, opts) return resultCh } // Return true if the character is within the allowed characters in an XML 1.0 document // The list of allowed characters can be found here: https://www.w3.org/TR/xml/#charsets func validXMLChar(r rune) (ok bool) { return r == 0x09 || r == 0x0A || r == 0x0D || r >= 0x20 && r <= 0xD7FF || r >= 0xE000 && r <= 0xFFFD || r >= 0x10000 && r <= 0x10FFFF } func hasInvalidXMLChar(str string) bool { for _, s := range str { if !validXMLChar(s) { return true } } return false } // Generate and call MultiDelete S3 requests based on entries received from the iterator. func (c *Client) removeObjectsIter(ctx context.Context, bucketName string, objectsIter iter.Seq[ObjectInfo], yield func(RemoveObjectResult) bool, opts RemoveObjectsOptions) { maxEntries := 1000 urlValues := make(url.Values) urlValues.Set("delete", "") // Build headers. headers := make(http.Header) if opts.GovernanceBypass { // Set the bypass goverenance retention header headers.Set(amzBypassGovernance, "true") } processRemoveMultiObjectsResponseIter := func(batch []ObjectInfo, yield func(RemoveObjectResult) bool) bool { if len(batch) == 0 { return false } // Generate remove multi objects XML request removeBytes := generateRemoveMultiObjectsRequest(batch) // Execute POST on bucket to remove objects. resp, err := c.executeMethod(ctx, http.MethodPost, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentBody: bytes.NewReader(removeBytes), contentLength: int64(len(removeBytes)), contentMD5Base64: sumMD5Base64(removeBytes), contentSHA256Hex: sum256Hex(removeBytes), customHeader: headers, }) if resp != nil { defer closeResponse(resp) if resp.StatusCode != http.StatusOK { err = httpRespToErrorResponse(resp, bucketName, "") } } if err != nil { for _, b := range batch { if !yield(RemoveObjectResult{ ObjectName: b.Key, ObjectVersionID: b.VersionID, Err: err, }) { return false } } return false } // Parse multi delete XML response rmResult := &deleteMultiObjectsResult{} if err := xmlDecoder(resp.Body, rmResult); err != nil { yield(RemoveObjectResult{ObjectName: "", Err: err}) return false } // Fill deletion that returned an error. for _, obj := range rmResult.UnDeletedObjects { // Version does not exist is not an error ignore and continue. switch obj.Code { case "InvalidArgument", "NoSuchVersion": continue } if !yield(RemoveObjectResult{ ObjectName: obj.Key, ObjectVersionID: obj.VersionID, Err: ErrorResponse{ Code: obj.Code, Message: obj.Message, }, }) { return false } } // Fill deletion that returned success for _, obj := range rmResult.DeletedObjects { if !yield(RemoveObjectResult{ ObjectName: obj.Key, // Only filled with versioned buckets ObjectVersionID: obj.VersionID, DeleteMarker: obj.DeleteMarker, DeleteMarkerVersionID: obj.DeleteMarkerVersionID, }) { return false } } return true } var batch []ObjectInfo next, stop := iter.Pull(objectsIter) defer stop() for { // Loop over entries by 1000 and call MultiDelete requests object, ok := next() if !ok { // delete the remaining batch. processRemoveMultiObjectsResponseIter(batch, yield) return } if hasInvalidXMLChar(object.Key) { // Use single DELETE so the object name will be in the request URL instead of the multi-delete XML document. removeResult := c.removeObject(ctx, bucketName, object.Key, RemoveObjectOptions{ VersionID: object.VersionID, GovernanceBypass: opts.GovernanceBypass, }) if err := removeResult.Err; err != nil { // Version does not exist is not an error ignore and continue. switch ToErrorResponse(err).Code { case "InvalidArgument", "NoSuchVersion": continue } } if !yield(removeResult) { return } continue } batch = append(batch, object) if len(batch) < maxEntries { continue } if !processRemoveMultiObjectsResponseIter(batch, yield) { return } batch = batch[:0] } } // Generate and call MultiDelete S3 requests based on entries received from objectsCh func (c *Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, resultCh chan<- RemoveObjectResult, opts RemoveObjectsOptions) { maxEntries := 1000 finish := false urlValues := make(url.Values) urlValues.Set("delete", "") // Close result channel when Multi delete finishes. defer close(resultCh) // Loop over entries by 1000 and call MultiDelete requests for !finish { count := 0 var batch []ObjectInfo // Try to gather 1000 entries for object := range objectsCh { if hasInvalidXMLChar(object.Key) { // Use single DELETE so the object name will be in the request URL instead of the multi-delete XML document. removeResult := c.removeObject(ctx, bucketName, object.Key, RemoveObjectOptions{ VersionID: object.VersionID, GovernanceBypass: opts.GovernanceBypass, }) if err := removeResult.Err; err != nil { // Version does not exist is not an error ignore and continue. switch ToErrorResponse(err).Code { case InvalidArgument, NoSuchVersion: continue } resultCh <- removeResult } resultCh <- removeResult continue } batch = append(batch, object) if count++; count >= maxEntries { break } } if count == 0 { // Multi Objects Delete API doesn't accept empty object list, quit immediately break } if count < maxEntries { // We didn't have 1000 entries, so this is the last batch finish = true } // Build headers. headers := make(http.Header) if opts.GovernanceBypass { // Set the bypass goverenance retention header headers.Set(amzBypassGovernance, "true") } // Generate remove multi objects XML request removeBytes := generateRemoveMultiObjectsRequest(batch) // Execute POST on bucket to remove objects. resp, err := c.executeMethod(ctx, http.MethodPost, requestMetadata{ bucketName: bucketName, queryValues: urlValues, contentBody: bytes.NewReader(removeBytes), contentLength: int64(len(removeBytes)), contentMD5Base64: sumMD5Base64(removeBytes), contentSHA256Hex: sum256Hex(removeBytes), customHeader: headers, expect200OKWithError: true, }) if resp != nil && resp.StatusCode != http.StatusOK { err = httpRespToErrorResponse(resp, bucketName, "") } if err != nil { for _, b := range batch { resultCh <- RemoveObjectResult{ ObjectName: b.Key, ObjectVersionID: b.VersionID, Err: err, } } continue } // Process multiobjects remove xml response processRemoveMultiObjectsResponse(resp.Body, resultCh) closeResponse(resp) } } // RemoveIncompleteUpload aborts an partially uploaded object. func (c *Client) RemoveIncompleteUpload(ctx context.Context, bucketName, objectName string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return err } // Find multipart upload ids of the object to be aborted. uploadIDs, err := c.findUploadIDs(ctx, bucketName, objectName) if err != nil { return err } for _, uploadID := range uploadIDs { // abort incomplete multipart upload, based on the upload id passed. err := c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) if err != nil { return err } } return nil } // abortMultipartUpload aborts a multipart upload for the given // uploadID, all previously uploaded parts are deleted. func (c *Client) abortMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return err } // Initialize url queries. urlValues := make(url.Values) urlValues.Set("uploadId", uploadID) // Execute DELETE on multipart upload. resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { return err } if resp != nil { if resp.StatusCode != http.StatusNoContent { // Abort has no response body, handle it for any errors. var errorResponse ErrorResponse switch resp.StatusCode { case http.StatusNotFound: // This is needed specifically for abort and it cannot // be converged into default case. errorResponse = ErrorResponse{ Code: NoSuchUpload, Message: "The specified multipart upload does not exist.", BucketName: bucketName, Key: objectName, RequestID: resp.Header.Get("x-amz-request-id"), HostID: resp.Header.Get("x-amz-id-2"), Region: resp.Header.Get("x-amz-bucket-region"), } default: return httpRespToErrorResponse(resp, bucketName, objectName) } return errorResponse } } return nil } minio-go-7.0.97/api-restore.go000066400000000000000000000132341510244170000161120ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * (C) 2018-2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/xml" "net/http" "net/url" "github.com/minio/minio-go/v7/pkg/s3utils" "github.com/minio/minio-go/v7/pkg/tags" ) // RestoreType represents the restore request type type RestoreType string const ( // RestoreSelect represents the restore SELECT operation RestoreSelect = RestoreType("SELECT") ) // TierType represents a retrieval tier type TierType string const ( // TierStandard is the standard retrieval tier TierStandard = TierType("Standard") // TierBulk is the bulk retrieval tier TierBulk = TierType("Bulk") // TierExpedited is the expedited retrieval tier TierExpedited = TierType("Expedited") ) // GlacierJobParameters represents the retrieval tier parameter type GlacierJobParameters struct { Tier TierType } // Encryption contains the type of server-side encryption used during object retrieval type Encryption struct { EncryptionType string KMSContext string KMSKeyID string `xml:"KMSKeyId"` } // MetadataEntry represents a metadata information of the restored object. type MetadataEntry struct { Name string Value string } // S3 holds properties of the copy of the archived object type S3 struct { AccessControlList *AccessControlList `xml:"AccessControlList,omitempty"` BucketName string Prefix string CannedACL *string `xml:"CannedACL,omitempty"` Encryption *Encryption `xml:"Encryption,omitempty"` StorageClass *string `xml:"StorageClass,omitempty"` Tagging *tags.Tags `xml:"Tagging,omitempty"` UserMetadata *MetadataEntry `xml:"UserMetadata,omitempty"` } // SelectParameters holds the select request parameters type SelectParameters struct { XMLName xml.Name `xml:"SelectParameters"` ExpressionType QueryExpressionType Expression string InputSerialization SelectObjectInputSerialization OutputSerialization SelectObjectOutputSerialization } // OutputLocation holds properties of the copy of the archived object type OutputLocation struct { XMLName xml.Name `xml:"OutputLocation"` S3 S3 `xml:"S3"` } // RestoreRequest holds properties of the restore object request type RestoreRequest struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ RestoreRequest"` Type *RestoreType `xml:"Type,omitempty"` Tier *TierType `xml:"Tier,omitempty"` Days *int `xml:"Days,omitempty"` GlacierJobParameters *GlacierJobParameters `xml:"GlacierJobParameters,omitempty"` Description *string `xml:"Description,omitempty"` SelectParameters *SelectParameters `xml:"SelectParameters,omitempty"` OutputLocation *OutputLocation `xml:"OutputLocation,omitempty"` } // SetDays sets the days parameter of the restore request func (r *RestoreRequest) SetDays(v int) { r.Days = &v } // SetGlacierJobParameters sets the GlacierJobParameters of the restore request func (r *RestoreRequest) SetGlacierJobParameters(v GlacierJobParameters) { r.GlacierJobParameters = &v } // SetType sets the type of the restore request func (r *RestoreRequest) SetType(v RestoreType) { r.Type = &v } // SetTier sets the retrieval tier of the restore request func (r *RestoreRequest) SetTier(v TierType) { r.Tier = &v } // SetDescription sets the description of the restore request func (r *RestoreRequest) SetDescription(v string) { r.Description = &v } // SetSelectParameters sets SelectParameters of the restore select request func (r *RestoreRequest) SetSelectParameters(v SelectParameters) { r.SelectParameters = &v } // SetOutputLocation sets the properties of the copy of the archived object func (r *RestoreRequest) SetOutputLocation(v OutputLocation) { r.OutputLocation = &v } // RestoreObject is a implementation of https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html AWS S3 API func (c *Client) RestoreObject(ctx context.Context, bucketName, objectName, versionID string, req RestoreRequest) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return err } restoreRequestBytes, err := xml.Marshal(req) if err != nil { return err } urlValues := make(url.Values) urlValues.Set("restore", "") if versionID != "" { urlValues.Set("versionId", versionID) } // Execute POST on bucket/object. resp, err := c.executeMethod(ctx, http.MethodPost, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, contentMD5Base64: sumMD5Base64(restoreRequestBytes), contentSHA256Hex: sum256Hex(restoreRequestBytes), contentBody: bytes.NewReader(restoreRequestBytes), contentLength: int64(len(restoreRequestBytes)), }) defer closeResponse(resp) if err != nil { return err } if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK { return httpRespToErrorResponse(resp, bucketName, "") } return nil } minio-go-7.0.97/api-s3-datatypes.go000066400000000000000000000317521510244170000167550ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "encoding/base64" "encoding/xml" "errors" "io" "reflect" "time" ) // listAllMyBucketsResult container for listBuckets response. type listAllMyBucketsResult struct { // Container for one or more buckets. Buckets struct { Bucket []BucketInfo } Owner owner } // listAllMyDirectoryBucketsResult container for listDirectoryBuckets response. type listAllMyDirectoryBucketsResult struct { Buckets struct { Bucket []BucketInfo } ContinuationToken string } // owner container for bucket owner information. type owner struct { DisplayName string ID string } // CommonPrefix container for prefix response. type CommonPrefix struct { Prefix string } // ListBucketV2Result container for listObjects response version 2. type ListBucketV2Result struct { // A response can contain CommonPrefixes only if you have // specified a delimiter. CommonPrefixes []CommonPrefix // Metadata about each object returned. Contents []ObjectInfo Delimiter string // Encoding type used to encode object keys in the response. EncodingType string // A flag that indicates whether or not ListObjects returned all of the results // that satisfied the search criteria. IsTruncated bool MaxKeys int64 Name string // Hold the token that will be sent in the next request to fetch the next group of keys NextContinuationToken string ContinuationToken string Prefix string // FetchOwner and StartAfter are currently not used FetchOwner string StartAfter string } // Version is an element in the list object versions response type Version struct { ETag string IsLatest bool Key string LastModified time.Time Owner Owner Size int64 StorageClass string VersionID string `xml:"VersionId"` // x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value. // Only returned by MinIO servers. UserMetadata StringMap `json:"userMetadata,omitempty"` // x-amz-tagging values in their k/v values. // Only returned by MinIO servers. UserTags URLMap `json:"userTags,omitempty" xml:"UserTags"` Internal *struct { K int // Data blocks M int // Parity blocks } `xml:"Internal"` // Checksum values. Only returned by AiStor servers. ChecksumCRC32 string `xml:",omitempty"` ChecksumCRC32C string `xml:",omitempty"` ChecksumSHA1 string `xml:",omitempty"` ChecksumSHA256 string `xml:",omitempty"` ChecksumCRC64NVME string `xml:",omitempty"` ChecksumType string `xml:",omitempty"` isDeleteMarker bool } // ListVersionsResult is an element in the list object versions response // and has a special Unmarshaler because we need to preserver the order // of and in ListVersionsResult.Versions slice type ListVersionsResult struct { Versions []Version CommonPrefixes []CommonPrefix Name string Prefix string Delimiter string MaxKeys int64 EncodingType string IsTruncated bool KeyMarker string VersionIDMarker string NextKeyMarker string NextVersionIDMarker string } // UnmarshalXML is a custom unmarshal code for the response of ListObjectVersions, the custom // code will unmarshal and tags and save them in Versions field to // preserve the lexical order of the listing. func (l *ListVersionsResult) UnmarshalXML(d *xml.Decoder, _ xml.StartElement) (err error) { for { // Read tokens from the XML document in a stream. t, err := d.Token() if err != nil { if err == io.EOF { break } return err } se, ok := t.(xml.StartElement) if ok { tagName := se.Name.Local switch tagName { case "Name", "Prefix", "Delimiter", "EncodingType", "KeyMarker", "NextKeyMarker": var s string if err = d.DecodeElement(&s, &se); err != nil { return err } v := reflect.ValueOf(l).Elem().FieldByName(tagName) if v.IsValid() { v.SetString(s) } case "VersionIdMarker": // VersionIdMarker is a special case because of 'Id' instead of 'ID' in field name var s string if err = d.DecodeElement(&s, &se); err != nil { return err } l.VersionIDMarker = s case "NextVersionIdMarker": // NextVersionIdMarker is a special case because of 'Id' instead of 'ID' in field name var s string if err = d.DecodeElement(&s, &se); err != nil { return err } l.NextVersionIDMarker = s case "IsTruncated": // bool var b bool if err = d.DecodeElement(&b, &se); err != nil { return err } l.IsTruncated = b case "MaxKeys": // int64 var i int64 if err = d.DecodeElement(&i, &se); err != nil { return err } l.MaxKeys = i case "CommonPrefixes": var cp CommonPrefix if err = d.DecodeElement(&cp, &se); err != nil { return err } l.CommonPrefixes = append(l.CommonPrefixes, cp) case "DeleteMarker", "Version": var v Version if err = d.DecodeElement(&v, &se); err != nil { return err } if tagName == "DeleteMarker" { v.isDeleteMarker = true } l.Versions = append(l.Versions, v) default: return errors.New("unrecognized option:" + tagName) } } } return nil } // ListBucketResult container for listObjects response. type ListBucketResult struct { // A response can contain CommonPrefixes only if you have // specified a delimiter. CommonPrefixes []CommonPrefix // Metadata about each object returned. Contents []ObjectInfo Delimiter string // Encoding type used to encode object keys in the response. EncodingType string // A flag that indicates whether or not ListObjects returned all of the results // that satisfied the search criteria. IsTruncated bool Marker string MaxKeys int64 Name string // When response is truncated (the IsTruncated element value in // the response is true), you can use the key name in this field // as marker in the subsequent request to get next set of objects. // Object storage lists objects in alphabetical order Note: This // element is returned only if you have delimiter request // parameter specified. If response does not include the NextMaker // and it is truncated, you can use the value of the last Key in // the response as the marker in the subsequent request to get the // next set of object keys. NextMarker string Prefix string } // ListMultipartUploadsResult container for ListMultipartUploads response type ListMultipartUploadsResult struct { Bucket string KeyMarker string UploadIDMarker string `xml:"UploadIdMarker"` NextKeyMarker string NextUploadIDMarker string `xml:"NextUploadIdMarker"` EncodingType string MaxUploads int64 IsTruncated bool Uploads []ObjectMultipartInfo `xml:"Upload"` Prefix string Delimiter string // A response can contain CommonPrefixes only if you specify a delimiter. CommonPrefixes []CommonPrefix } // initiator container for who initiated multipart upload. type initiator struct { ID string DisplayName string } // copyObjectResult container for copy object response. type copyObjectResult struct { ETag string LastModified time.Time // time string format "2006-01-02T15:04:05.000Z" } // ObjectPart container for particular part of an object. type ObjectPart struct { // Part number identifies the part. PartNumber int // Date and time the part was uploaded. LastModified time.Time // Entity tag returned when the part was uploaded, usually md5sum // of the part. ETag string // Size of the uploaded part data. Size int64 // Checksum values of each part. ChecksumCRC32 string ChecksumCRC32C string ChecksumSHA1 string ChecksumSHA256 string ChecksumCRC64NVME string } // Checksum will return the checksum for the given type. // Will return the empty string if not set. func (c ObjectPart) Checksum(t ChecksumType) string { switch { case t.Is(ChecksumCRC32C): return c.ChecksumCRC32C case t.Is(ChecksumCRC32): return c.ChecksumCRC32 case t.Is(ChecksumSHA1): return c.ChecksumSHA1 case t.Is(ChecksumSHA256): return c.ChecksumSHA256 case t.Is(ChecksumCRC64NVME): return c.ChecksumCRC64NVME } return "" } // ChecksumRaw returns the decoded checksum from the part. func (c ObjectPart) ChecksumRaw(t ChecksumType) ([]byte, error) { b64 := c.Checksum(t) if b64 == "" { return nil, errors.New("no checksum set") } decoded, err := base64.StdEncoding.DecodeString(b64) if err != nil { return nil, err } if len(decoded) != t.RawByteLen() { return nil, errors.New("checksum length mismatch") } return decoded, nil } // ListObjectPartsResult container for ListObjectParts response. type ListObjectPartsResult struct { Bucket string Key string UploadID string `xml:"UploadId"` Initiator initiator Owner owner StorageClass string PartNumberMarker int NextPartNumberMarker int MaxParts int // ChecksumAlgorithm will be CRC32, CRC32C, etc. ChecksumAlgorithm string // ChecksumType is FULL_OBJECT or COMPOSITE (assume COMPOSITE when unset) ChecksumType string // Indicates whether the returned list of parts is truncated. IsTruncated bool ObjectParts []ObjectPart `xml:"Part"` EncodingType string } // initiateMultipartUploadResult container for InitiateMultiPartUpload // response. type initiateMultipartUploadResult struct { Bucket string Key string UploadID string `xml:"UploadId"` } // completeMultipartUploadResult container for completed multipart // upload response. type completeMultipartUploadResult struct { Location string Bucket string Key string ETag string // Checksum values, hash of hashes of parts. ChecksumCRC32 string ChecksumCRC32C string ChecksumSHA1 string ChecksumSHA256 string ChecksumCRC64NVME string ChecksumType string } // CompletePart sub container lists individual part numbers and their // md5sum, part of completeMultipartUpload. type CompletePart struct { // Part number identifies the part. PartNumber int ETag string // Checksum values ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"` ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"` ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"` ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"` ChecksumCRC64NVME string `xml:",omitempty"` } // Checksum will return the checksum for the given type. // Will return the empty string if not set. func (c CompletePart) Checksum(t ChecksumType) string { switch { case t.Is(ChecksumCRC32C): return c.ChecksumCRC32C case t.Is(ChecksumCRC32): return c.ChecksumCRC32 case t.Is(ChecksumSHA1): return c.ChecksumSHA1 case t.Is(ChecksumSHA256): return c.ChecksumSHA256 case t.Is(ChecksumCRC64NVME): return c.ChecksumCRC64NVME } return "" } // completeMultipartUpload container for completing multipart upload. type completeMultipartUpload struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUpload" json:"-"` Parts []CompletePart `xml:"Part"` } // createBucketConfiguration container for bucket configuration. type createBucketConfiguration struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreateBucketConfiguration" json:"-"` Location string `xml:"LocationConstraint"` } // deleteObject container for Delete element in MultiObjects Delete XML request type deleteObject struct { Key string VersionID string `xml:"VersionId,omitempty"` } // deletedObject container for Deleted element in MultiObjects Delete XML response type deletedObject struct { Key string VersionID string `xml:"VersionId,omitempty"` // These fields are ignored. DeleteMarker bool DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId,omitempty"` } // nonDeletedObject container for Error element (failed deletion) in MultiObjects Delete XML response type nonDeletedObject struct { Key string Code string Message string VersionID string `xml:"VersionId"` } // deletedMultiObjects container for MultiObjects Delete XML request type deleteMultiObjects struct { XMLName xml.Name `xml:"Delete"` Quiet bool Objects []deleteObject `xml:"Object"` } // deletedMultiObjectsResult container for MultiObjects Delete XML response type deleteMultiObjectsResult struct { XMLName xml.Name `xml:"DeleteResult"` DeletedObjects []deletedObject `xml:"Deleted"` UnDeletedObjects []nonDeletedObject `xml:"Error"` } minio-go-7.0.97/api-select.go000066400000000000000000000521511510244170000157070ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * (C) 2018-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/binary" "encoding/xml" "errors" "fmt" "hash" "hash/crc32" "io" "net/http" "net/url" "strings" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/s3utils" ) // CSVFileHeaderInfo - is the parameter for whether to utilize headers. type CSVFileHeaderInfo string // Constants for file header info. const ( CSVFileHeaderInfoNone CSVFileHeaderInfo = "NONE" CSVFileHeaderInfoIgnore CSVFileHeaderInfo = "IGNORE" CSVFileHeaderInfoUse CSVFileHeaderInfo = "USE" ) // SelectCompressionType - is the parameter for what type of compression is // present type SelectCompressionType string // Constants for compression types under select API. const ( SelectCompressionNONE SelectCompressionType = "NONE" SelectCompressionGZIP SelectCompressionType = "GZIP" SelectCompressionBZIP SelectCompressionType = "BZIP2" // Non-standard compression schemes, supported by MinIO hosts: SelectCompressionZSTD SelectCompressionType = "ZSTD" // Zstandard compression. SelectCompressionLZ4 SelectCompressionType = "LZ4" // LZ4 Stream SelectCompressionS2 SelectCompressionType = "S2" // S2 Stream SelectCompressionSNAPPY SelectCompressionType = "SNAPPY" // Snappy stream ) // CSVQuoteFields - is the parameter for how CSV fields are quoted. type CSVQuoteFields string // Constants for csv quote styles. const ( CSVQuoteFieldsAlways CSVQuoteFields = "Always" CSVQuoteFieldsAsNeeded CSVQuoteFields = "AsNeeded" ) // QueryExpressionType - is of what syntax the expression is, this should only // be SQL type QueryExpressionType string // Constants for expression type. const ( QueryExpressionTypeSQL QueryExpressionType = "SQL" ) // JSONType determines json input serialization type. type JSONType string // Constants for JSONTypes. const ( JSONDocumentType JSONType = "DOCUMENT" JSONLinesType JSONType = "LINES" ) // ParquetInputOptions parquet input specific options type ParquetInputOptions struct{} // CSVInputOptions csv input specific options type CSVInputOptions struct { FileHeaderInfo CSVFileHeaderInfo fileHeaderInfoSet bool RecordDelimiter string recordDelimiterSet bool FieldDelimiter string fieldDelimiterSet bool QuoteCharacter string quoteCharacterSet bool QuoteEscapeCharacter string quoteEscapeCharacterSet bool Comments string commentsSet bool } // SetFileHeaderInfo sets the file header info in the CSV input options func (c *CSVInputOptions) SetFileHeaderInfo(val CSVFileHeaderInfo) { c.FileHeaderInfo = val c.fileHeaderInfoSet = true } // SetRecordDelimiter sets the record delimiter in the CSV input options func (c *CSVInputOptions) SetRecordDelimiter(val string) { c.RecordDelimiter = val c.recordDelimiterSet = true } // SetFieldDelimiter sets the field delimiter in the CSV input options func (c *CSVInputOptions) SetFieldDelimiter(val string) { c.FieldDelimiter = val c.fieldDelimiterSet = true } // SetQuoteCharacter sets the quote character in the CSV input options func (c *CSVInputOptions) SetQuoteCharacter(val string) { c.QuoteCharacter = val c.quoteCharacterSet = true } // SetQuoteEscapeCharacter sets the quote escape character in the CSV input options func (c *CSVInputOptions) SetQuoteEscapeCharacter(val string) { c.QuoteEscapeCharacter = val c.quoteEscapeCharacterSet = true } // SetComments sets the comments character in the CSV input options func (c *CSVInputOptions) SetComments(val string) { c.Comments = val c.commentsSet = true } // MarshalXML - produces the xml representation of the CSV input options struct func (c CSVInputOptions) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if err := e.EncodeToken(start); err != nil { return err } if c.FileHeaderInfo != "" || c.fileHeaderInfoSet { if err := e.EncodeElement(c.FileHeaderInfo, xml.StartElement{Name: xml.Name{Local: "FileHeaderInfo"}}); err != nil { return err } } if c.RecordDelimiter != "" || c.recordDelimiterSet { if err := e.EncodeElement(c.RecordDelimiter, xml.StartElement{Name: xml.Name{Local: "RecordDelimiter"}}); err != nil { return err } } if c.FieldDelimiter != "" || c.fieldDelimiterSet { if err := e.EncodeElement(c.FieldDelimiter, xml.StartElement{Name: xml.Name{Local: "FieldDelimiter"}}); err != nil { return err } } if c.QuoteCharacter != "" || c.quoteCharacterSet { if err := e.EncodeElement(c.QuoteCharacter, xml.StartElement{Name: xml.Name{Local: "QuoteCharacter"}}); err != nil { return err } } if c.QuoteEscapeCharacter != "" || c.quoteEscapeCharacterSet { if err := e.EncodeElement(c.QuoteEscapeCharacter, xml.StartElement{Name: xml.Name{Local: "QuoteEscapeCharacter"}}); err != nil { return err } } if c.Comments != "" || c.commentsSet { if err := e.EncodeElement(c.Comments, xml.StartElement{Name: xml.Name{Local: "Comments"}}); err != nil { return err } } return e.EncodeToken(xml.EndElement{Name: start.Name}) } // CSVOutputOptions csv output specific options type CSVOutputOptions struct { QuoteFields CSVQuoteFields quoteFieldsSet bool RecordDelimiter string recordDelimiterSet bool FieldDelimiter string fieldDelimiterSet bool QuoteCharacter string quoteCharacterSet bool QuoteEscapeCharacter string quoteEscapeCharacterSet bool } // SetQuoteFields sets the quote field parameter in the CSV output options func (c *CSVOutputOptions) SetQuoteFields(val CSVQuoteFields) { c.QuoteFields = val c.quoteFieldsSet = true } // SetRecordDelimiter sets the record delimiter character in the CSV output options func (c *CSVOutputOptions) SetRecordDelimiter(val string) { c.RecordDelimiter = val c.recordDelimiterSet = true } // SetFieldDelimiter sets the field delimiter character in the CSV output options func (c *CSVOutputOptions) SetFieldDelimiter(val string) { c.FieldDelimiter = val c.fieldDelimiterSet = true } // SetQuoteCharacter sets the quote character in the CSV output options func (c *CSVOutputOptions) SetQuoteCharacter(val string) { c.QuoteCharacter = val c.quoteCharacterSet = true } // SetQuoteEscapeCharacter sets the quote escape character in the CSV output options func (c *CSVOutputOptions) SetQuoteEscapeCharacter(val string) { c.QuoteEscapeCharacter = val c.quoteEscapeCharacterSet = true } // MarshalXML - produces the xml representation of the CSVOutputOptions struct func (c CSVOutputOptions) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if err := e.EncodeToken(start); err != nil { return err } if c.QuoteFields != "" || c.quoteFieldsSet { if err := e.EncodeElement(c.QuoteFields, xml.StartElement{Name: xml.Name{Local: "QuoteFields"}}); err != nil { return err } } if c.RecordDelimiter != "" || c.recordDelimiterSet { if err := e.EncodeElement(c.RecordDelimiter, xml.StartElement{Name: xml.Name{Local: "RecordDelimiter"}}); err != nil { return err } } if c.FieldDelimiter != "" || c.fieldDelimiterSet { if err := e.EncodeElement(c.FieldDelimiter, xml.StartElement{Name: xml.Name{Local: "FieldDelimiter"}}); err != nil { return err } } if c.QuoteCharacter != "" || c.quoteCharacterSet { if err := e.EncodeElement(c.QuoteCharacter, xml.StartElement{Name: xml.Name{Local: "QuoteCharacter"}}); err != nil { return err } } if c.QuoteEscapeCharacter != "" || c.quoteEscapeCharacterSet { if err := e.EncodeElement(c.QuoteEscapeCharacter, xml.StartElement{Name: xml.Name{Local: "QuoteEscapeCharacter"}}); err != nil { return err } } return e.EncodeToken(xml.EndElement{Name: start.Name}) } // JSONInputOptions json input specific options type JSONInputOptions struct { Type JSONType typeSet bool } // SetType sets the JSON type in the JSON input options func (j *JSONInputOptions) SetType(typ JSONType) { j.Type = typ j.typeSet = true } // MarshalXML - produces the xml representation of the JSONInputOptions struct func (j JSONInputOptions) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if err := e.EncodeToken(start); err != nil { return err } if j.Type != "" || j.typeSet { if err := e.EncodeElement(j.Type, xml.StartElement{Name: xml.Name{Local: "Type"}}); err != nil { return err } } return e.EncodeToken(xml.EndElement{Name: start.Name}) } // JSONOutputOptions - json output specific options type JSONOutputOptions struct { RecordDelimiter string recordDelimiterSet bool } // SetRecordDelimiter sets the record delimiter in the JSON output options func (j *JSONOutputOptions) SetRecordDelimiter(val string) { j.RecordDelimiter = val j.recordDelimiterSet = true } // MarshalXML - produces the xml representation of the JSONOutputOptions struct func (j JSONOutputOptions) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if err := e.EncodeToken(start); err != nil { return err } if j.RecordDelimiter != "" || j.recordDelimiterSet { if err := e.EncodeElement(j.RecordDelimiter, xml.StartElement{Name: xml.Name{Local: "RecordDelimiter"}}); err != nil { return err } } return e.EncodeToken(xml.EndElement{Name: start.Name}) } // SelectObjectInputSerialization - input serialization parameters type SelectObjectInputSerialization struct { CompressionType SelectCompressionType `xml:"CompressionType,omitempty"` Parquet *ParquetInputOptions `xml:"Parquet,omitempty"` CSV *CSVInputOptions `xml:"CSV,omitempty"` JSON *JSONInputOptions `xml:"JSON,omitempty"` } // SelectObjectOutputSerialization - output serialization parameters. type SelectObjectOutputSerialization struct { CSV *CSVOutputOptions `xml:"CSV,omitempty"` JSON *JSONOutputOptions `xml:"JSON,omitempty"` } // SelectObjectOptions - represents the input select body type SelectObjectOptions struct { XMLName xml.Name `xml:"SelectObjectContentRequest" json:"-"` ServerSideEncryption encrypt.ServerSide `xml:"-"` Expression string ExpressionType QueryExpressionType InputSerialization SelectObjectInputSerialization OutputSerialization SelectObjectOutputSerialization RequestProgress struct { Enabled bool } } // Header returns the http.Header representation of the SelectObject options. func (o SelectObjectOptions) Header() http.Header { headers := make(http.Header) if o.ServerSideEncryption != nil && o.ServerSideEncryption.Type() == encrypt.SSEC { o.ServerSideEncryption.Marshal(headers) } return headers } // SelectObjectType - is the parameter which defines what type of object the // operation is being performed on. type SelectObjectType string // Constants for input data types. const ( SelectObjectTypeCSV SelectObjectType = "CSV" SelectObjectTypeJSON SelectObjectType = "JSON" SelectObjectTypeParquet SelectObjectType = "Parquet" ) // preludeInfo is used for keeping track of necessary information from the // prelude. type preludeInfo struct { totalLen uint32 headerLen uint32 } // SelectResults is used for the streaming responses from the server. type SelectResults struct { pipeReader *io.PipeReader resp *http.Response stats *StatsMessage progress *ProgressMessage } // ProgressMessage is a struct for progress xml message. type ProgressMessage struct { XMLName xml.Name `xml:"Progress" json:"-"` StatsMessage } // StatsMessage is a struct for stat xml message. type StatsMessage struct { XMLName xml.Name `xml:"Stats" json:"-"` BytesScanned int64 BytesProcessed int64 BytesReturned int64 } // messageType represents the type of message. type messageType string const ( errorMsg messageType = "error" commonMsg messageType = "event" ) // eventType represents the type of event. type eventType string // list of event-types returned by Select API. const ( endEvent eventType = "End" recordsEvent eventType = "Records" progressEvent eventType = "Progress" statsEvent eventType = "Stats" ) // contentType represents content type of event. type contentType string const ( xmlContent contentType = "text/xml" ) // SelectObjectContent is a implementation of http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html AWS S3 API. func (c *Client) SelectObjectContent(ctx context.Context, bucketName, objectName string, opts SelectObjectOptions) (*SelectResults, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return nil, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { return nil, err } selectReqBytes, err := xml.Marshal(opts) if err != nil { return nil, err } urlValues := make(url.Values) urlValues.Set("select", "") urlValues.Set("select-type", "2") // Execute POST on bucket/object. resp, err := c.executeMethod(ctx, http.MethodPost, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: urlValues, customHeader: opts.Header(), contentMD5Base64: sumMD5Base64(selectReqBytes), contentSHA256Hex: sum256Hex(selectReqBytes), contentBody: bytes.NewReader(selectReqBytes), contentLength: int64(len(selectReqBytes)), }) if err != nil { return nil, err } return NewSelectResults(resp, bucketName) } // NewSelectResults creates a Select Result parser that parses the response // and returns a Reader that will return parsed and assembled select output. func NewSelectResults(resp *http.Response, bucketName string) (*SelectResults, error) { if resp.StatusCode != http.StatusOK { return nil, httpRespToErrorResponse(resp, bucketName, "") } pipeReader, pipeWriter := io.Pipe() streamer := &SelectResults{ resp: resp, stats: &StatsMessage{}, progress: &ProgressMessage{}, pipeReader: pipeReader, } streamer.start(pipeWriter) return streamer, nil } // Close - closes the underlying response body and the stream reader. func (s *SelectResults) Close() error { defer closeResponse(s.resp) return s.pipeReader.Close() } // Read - is a reader compatible implementation for SelectObjectContent records. func (s *SelectResults) Read(b []byte) (n int, err error) { return s.pipeReader.Read(b) } // Stats - information about a request's stats when processing is complete. func (s *SelectResults) Stats() *StatsMessage { return s.stats } // Progress - information about the progress of a request. func (s *SelectResults) Progress() *ProgressMessage { return s.progress } // start is the main function that decodes the large byte array into // several events that are sent through the eventstream. func (s *SelectResults) start(pipeWriter *io.PipeWriter) { go func() { for { var prelude preludeInfo headers := make(http.Header) var err error // Create CRC code crc := crc32.New(crc32.IEEETable) crcReader := io.TeeReader(s.resp.Body, crc) // Extract the prelude(12 bytes) into a struct to extract relevant information. prelude, err = processPrelude(crcReader, crc) if err != nil { pipeWriter.CloseWithError(err) closeResponse(s.resp) return } // Extract the headers(variable bytes) into a struct to extract relevant information if prelude.headerLen > 0 { if err = extractHeader(io.LimitReader(crcReader, int64(prelude.headerLen)), headers); err != nil { pipeWriter.CloseWithError(err) closeResponse(s.resp) return } } // Get the actual payload length so that the appropriate amount of // bytes can be read or parsed. payloadLen := prelude.PayloadLen() m := messageType(headers.Get("message-type")) switch m { case errorMsg: pipeWriter.CloseWithError(errors.New(headers.Get("error-code") + ":\"" + headers.Get("error-message") + "\"")) closeResponse(s.resp) return case commonMsg: // Get content-type of the payload. c := contentType(headers.Get("content-type")) // Get event type of the payload. e := eventType(headers.Get("event-type")) // Handle all supported events. switch e { case endEvent: pipeWriter.Close() closeResponse(s.resp) return case recordsEvent: if _, err = io.Copy(pipeWriter, io.LimitReader(crcReader, payloadLen)); err != nil { pipeWriter.CloseWithError(err) closeResponse(s.resp) return } case progressEvent: switch c { case xmlContent: if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.progress); err != nil { pipeWriter.CloseWithError(err) closeResponse(s.resp) return } default: pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, progressEvent)) closeResponse(s.resp) return } case statsEvent: switch c { case xmlContent: if err = xmlDecoder(io.LimitReader(crcReader, payloadLen), s.stats); err != nil { pipeWriter.CloseWithError(err) closeResponse(s.resp) return } default: pipeWriter.CloseWithError(fmt.Errorf("Unexpected content-type %s sent for event-type %s", c, statsEvent)) closeResponse(s.resp) return } } } // Ensures that the full message's CRC is correct and // that the message is not corrupted if err := checkCRC(s.resp.Body, crc.Sum32()); err != nil { pipeWriter.CloseWithError(err) closeResponse(s.resp) return } } }() } // PayloadLen is a function that calculates the length of the payload. func (p preludeInfo) PayloadLen() int64 { return int64(p.totalLen - p.headerLen - 16) } // processPrelude is the function that reads the 12 bytes of the prelude and // ensures the CRC is correct while also extracting relevant information into // the struct, func processPrelude(prelude io.Reader, crc hash.Hash32) (preludeInfo, error) { var err error pInfo := preludeInfo{} // reads total length of the message (first 4 bytes) pInfo.totalLen, err = extractUint32(prelude) if err != nil { return pInfo, err } // reads total header length of the message (2nd 4 bytes) pInfo.headerLen, err = extractUint32(prelude) if err != nil { return pInfo, err } // checks that the CRC is correct (3rd 4 bytes) preCRC := crc.Sum32() if err := checkCRC(prelude, preCRC); err != nil { return pInfo, err } return pInfo, nil } // extracts the relevant information from the Headers. func extractHeader(body io.Reader, myHeaders http.Header) error { for { // extracts the first part of the header, headerTypeName, err := extractHeaderType(body) if err != nil { // Since end of file, we have read all of our headers if err == io.EOF { break } return err } // reads the 7 present in the header and ignores it. extractUint8(body) headerValueName, err := extractHeaderValue(body) if err != nil { return err } myHeaders.Set(headerTypeName, headerValueName) } return nil } // extractHeaderType extracts the first half of the header message, the header type. func extractHeaderType(body io.Reader) (string, error) { // extracts 2 bit integer headerNameLen, err := extractUint8(body) if err != nil { return "", err } // extracts the string with the appropriate number of bytes headerName, err := extractString(body, int(headerNameLen)) if err != nil { return "", err } return strings.TrimPrefix(headerName, ":"), nil } // extractsHeaderValue extracts the second half of the header message, the // header value func extractHeaderValue(body io.Reader) (string, error) { bodyLen, err := extractUint16(body) if err != nil { return "", err } bodyName, err := extractString(body, int(bodyLen)) if err != nil { return "", err } return bodyName, nil } // extracts a string from byte array of a particular number of bytes. func extractString(source io.Reader, lenBytes int) (string, error) { myVal := make([]byte, lenBytes) _, err := source.Read(myVal) if err != nil { return "", err } return string(myVal), nil } // extractUint32 extracts a 4 byte integer from the byte array. func extractUint32(r io.Reader) (uint32, error) { buf := make([]byte, 4) _, err := readFull(r, buf) if err != nil { return 0, err } return binary.BigEndian.Uint32(buf), nil } // extractUint16 extracts a 2 byte integer from the byte array. func extractUint16(r io.Reader) (uint16, error) { buf := make([]byte, 2) _, err := readFull(r, buf) if err != nil { return 0, err } return binary.BigEndian.Uint16(buf), nil } // extractUint8 extracts a 1 byte integer from the byte array. func extractUint8(r io.Reader) (uint8, error) { buf := make([]byte, 1) _, err := readFull(r, buf) if err != nil { return 0, err } return buf[0], nil } // checkCRC ensures that the CRC matches with the one from the reader. func checkCRC(r io.Reader, expect uint32) error { msgCRC, err := extractUint32(r) if err != nil { return err } if msgCRC != expect { return fmt.Errorf("Checksum Mismatch, MessageCRC of 0x%X does not equal expected CRC of 0x%X", msgCRC, expect) } return nil } minio-go-7.0.97/api-stat.go000066400000000000000000000076231510244170000154070ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "net/http" "github.com/minio/minio-go/v7/pkg/s3utils" ) // BucketExists verifies if bucket exists and you have permission to access it. Allows for a Context to // control cancellations and timeouts. func (c *Client) BucketExists(ctx context.Context, bucketName string) (bool, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return false, err } // Execute HEAD on bucketName. resp, err := c.executeMethod(ctx, http.MethodHead, requestMetadata{ bucketName: bucketName, contentSHA256Hex: emptySHA256Hex, }) defer closeResponse(resp) if err != nil { if ToErrorResponse(err).Code == NoSuchBucket { return false, nil } return false, err } if resp != nil { resperr := httpRespToErrorResponse(resp, bucketName, "") if ToErrorResponse(resperr).Code == NoSuchBucket { return false, nil } if resp.StatusCode != http.StatusOK { return false, httpRespToErrorResponse(resp, bucketName, "") } } return true, nil } // StatObject verifies if object exists, you have permission to access it // and returns information about the object. func (c *Client) StatObject(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ObjectInfo{}, ErrorResponse{ StatusCode: http.StatusBadRequest, Code: InvalidBucketName, Message: err.Error(), } } if err := s3utils.CheckValidObjectName(objectName); err != nil { return ObjectInfo{}, ErrorResponse{ StatusCode: http.StatusBadRequest, Code: XMinioInvalidObjectName, Message: err.Error(), } } headers := opts.Header() if opts.Internal.ReplicationDeleteMarker { headers.Set(minIOBucketReplicationDeleteMarker, "true") } if opts.Internal.IsReplicationReadyForDeleteMarker { headers.Set(isMinioTgtReplicationReady, "true") } // Execute HEAD on objectName. resp, err := c.executeMethod(ctx, http.MethodHead, requestMetadata{ bucketName: bucketName, objectName: objectName, queryValues: opts.toQueryValues(), contentSHA256Hex: emptySHA256Hex, customHeader: headers, }) defer closeResponse(resp) if err != nil { return ObjectInfo{}, err } if resp != nil { deleteMarker := resp.Header.Get(amzDeleteMarker) == "true" replicationReady := resp.Header.Get(minioTgtReplicationReady) == "true" if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { if resp.StatusCode == http.StatusMethodNotAllowed && opts.VersionID != "" && deleteMarker { errResp := ErrorResponse{ StatusCode: resp.StatusCode, Code: MethodNotAllowed, Message: s3ErrorResponseMap[MethodNotAllowed], BucketName: bucketName, Key: objectName, } return ObjectInfo{ VersionID: resp.Header.Get(amzVersionID), IsDeleteMarker: deleteMarker, }, errResp } return ObjectInfo{ VersionID: resp.Header.Get(amzVersionID), IsDeleteMarker: deleteMarker, ReplicationReady: replicationReady, // whether delete marker can be replicated }, httpRespToErrorResponse(resp, bucketName, objectName) } } return ToObjectInfo(bucketName, objectName, resp.Header) } minio-go-7.0.97/api.go000066400000000000000000001046561510244170000144420ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2024 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/base64" "errors" "fmt" "io" "math/rand" "net" "net/http" "net/http/cookiejar" "net/http/httptrace" "net/http/httputil" "net/url" "os" "runtime" "strings" "sync" "sync/atomic" "time" "github.com/dustin/go-humanize" md5simd "github.com/minio/md5-simd" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/kvcache" "github.com/minio/minio-go/v7/pkg/s3utils" "github.com/minio/minio-go/v7/pkg/signer" "github.com/minio/minio-go/v7/pkg/singleflight" "golang.org/x/net/publicsuffix" ) // Client implements Amazon S3 compatible methods. type Client struct { // Standard options. // Parsed endpoint url provided by the user. endpointURL *url.URL // Holds various credential providers. credsProvider *credentials.Credentials // Custom signerType value overrides all credentials. overrideSignerType credentials.SignatureType // User supplied. appInfo struct { appName string appVersion string } // Indicate whether we are using https or not secure bool // Needs allocation. httpClient *http.Client httpTrace *httptrace.ClientTrace bucketLocCache *kvcache.Cache[string, string] bucketSessionCache *kvcache.Cache[string, credentials.Value] credsGroup singleflight.Group[string, credentials.Value] // Advanced functionality. isTraceEnabled bool traceErrorsOnly bool traceOutput io.Writer // S3 specific accelerated endpoint. s3AccelerateEndpoint string // S3 dual-stack endpoints are enabled by default. s3DualstackEnabled bool // Region endpoint region string // Random seed. random *rand.Rand // lookup indicates type of url lookup supported by server. If not specified, // default to Auto. lookup BucketLookupType // lookupFn is a custom function to return URL lookup type supported by the server. lookupFn func(u url.URL, bucketName string) BucketLookupType // Factory for MD5 hash functions. md5Hasher func() md5simd.Hasher sha256Hasher func() md5simd.Hasher healthStatus int32 trailingHeaderSupport bool maxRetries int } // Options for New method type Options struct { Creds *credentials.Credentials Secure bool Transport http.RoundTripper Trace *httptrace.ClientTrace Region string BucketLookup BucketLookupType // Allows setting a custom region lookup based on URL pattern // not all URL patterns are covered by this library so if you // have a custom endpoints with many regions you can use this // function to perform region lookups appropriately. CustomRegionViaURL func(u url.URL) string // Provide a custom function that returns BucketLookupType based // on the input URL, this is just like s3utils.IsVirtualHostSupported() // function but allows users to provide their own implementation. // Once this is set it overrides all settings for opts.BucketLookup // if this function returns BucketLookupAuto then default detection // via s3utils.IsVirtualHostSupported() is used, otherwise the // function is expected to return appropriate value as expected for // the URL the user wishes to honor. // // BucketName is passed additionally for the caller to ensure // handle situations where `bucketNames` have multiple `.` separators // in such case HTTPs certs will not work properly for *. // wildcards, so you need to specifically handle these situations // and not return bucket as part of DNS since those requests may fail. // // For better understanding look at s3utils.IsVirtualHostSupported() // implementation. BucketLookupViaURL func(u url.URL, bucketName string) BucketLookupType // TrailingHeaders indicates server support of trailing headers. // Only supported for v4 signatures. TrailingHeaders bool // Custom hash routines. Leave nil to use standard. CustomMD5 func() md5simd.Hasher CustomSHA256 func() md5simd.Hasher // Number of times a request is retried. Defaults to 10 retries if this option is not configured. // Set to 1 to disable retries. MaxRetries int } // Global constants. const ( libraryName = "minio-go" libraryVersion = "v7.0.96" ) // User Agent should always following the below style. // Please open an issue to discuss any new changes here. // // MinIO (OS; ARCH) LIB/VER APP/VER const ( libraryUserAgentPrefix = "MinIO (" + runtime.GOOS + "; " + runtime.GOARCH + ") " libraryUserAgent = libraryUserAgentPrefix + libraryName + "/" + libraryVersion ) // BucketLookupType is type of url lookup supported by server. type BucketLookupType int // Different types of url lookup supported by the server.Initialized to BucketLookupAuto const ( BucketLookupAuto BucketLookupType = iota BucketLookupDNS BucketLookupPath ) // New - instantiate minio client with options func New(endpoint string, opts *Options) (*Client, error) { if opts == nil { return nil, errors.New("no options provided") } clnt, err := privateNew(endpoint, opts) if err != nil { return nil, err } if s3utils.IsAmazonEndpoint(*clnt.endpointURL) { // If Amazon S3 set to signature v4. clnt.overrideSignerType = credentials.SignatureV4 // Amazon S3 endpoints are resolved into dual-stack endpoints by default // for backwards compatibility. clnt.s3DualstackEnabled = true } return clnt, nil } // EndpointURL returns the URL of the S3-compatible endpoint that this client connects to. // // Returns a copy of the endpoint URL to prevent modification of internal state. func (c *Client) EndpointURL() *url.URL { endpoint := *c.endpointURL // copy to prevent callers from modifying internal state return &endpoint } // lockedRandSource provides protected rand source, implements rand.Source interface. type lockedRandSource struct { lk sync.Mutex src rand.Source } // Int63 returns a non-negative pseudo-random 63-bit integer as an int64. func (r *lockedRandSource) Int63() (n int64) { r.lk.Lock() n = r.src.Int63() r.lk.Unlock() return n } // Seed uses the provided seed value to initialize the generator to a // deterministic state. func (r *lockedRandSource) Seed(seed int64) { r.lk.Lock() r.src.Seed(seed) r.lk.Unlock() } func privateNew(endpoint string, opts *Options) (*Client, error) { // construct endpoint. endpointURL, err := getEndpointURL(endpoint, opts.Secure) if err != nil { return nil, err } // Initialize cookies to preserve server sent cookies if any and replay // them upon each request. jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) if err != nil { return nil, err } // instantiate new Client. clnt := new(Client) // Save the credentials. clnt.credsProvider = opts.Creds // Remember whether we are using https or not clnt.secure = opts.Secure // Save endpoint URL, user agent for future uses. clnt.endpointURL = endpointURL transport := opts.Transport if transport == nil { transport, err = DefaultTransport(opts.Secure) if err != nil { return nil, err } } clnt.httpTrace = opts.Trace // Instantiate http client and bucket location cache. clnt.httpClient = &http.Client{ Jar: jar, Transport: transport, CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse }, } // Sets custom region, if region is empty bucket location cache is used automatically. if opts.Region == "" { if opts.CustomRegionViaURL != nil { opts.Region = opts.CustomRegionViaURL(*clnt.endpointURL) } else { opts.Region = s3utils.GetRegionFromURL(*clnt.endpointURL) } } clnt.region = opts.Region // Initialize bucket region cache. clnt.bucketLocCache = &kvcache.Cache[string, string]{} // Initialize bucket session cache (s3 express). clnt.bucketSessionCache = &kvcache.Cache[string, credentials.Value]{} // Introduce a new locked random seed. clnt.random = rand.New(&lockedRandSource{src: rand.NewSource(time.Now().UTC().UnixNano())}) // Add default md5 hasher. clnt.md5Hasher = opts.CustomMD5 clnt.sha256Hasher = opts.CustomSHA256 if clnt.md5Hasher == nil { clnt.md5Hasher = newMd5Hasher } if clnt.sha256Hasher == nil { clnt.sha256Hasher = newSHA256Hasher } clnt.trailingHeaderSupport = opts.TrailingHeaders && clnt.overrideSignerType.IsV4() // Sets bucket lookup style, whether server accepts DNS or Path lookup. Default is Auto - determined // by the SDK. When Auto is specified, DNS lookup is used for Amazon/Google cloud endpoints and Path for all other endpoints. clnt.lookup = opts.BucketLookup clnt.lookupFn = opts.BucketLookupViaURL // healthcheck is not initialized clnt.healthStatus = unknown clnt.maxRetries = MaxRetry if opts.MaxRetries > 0 { clnt.maxRetries = opts.MaxRetries } // Return. return clnt, nil } // SetAppInfo adds custom application name and version to the User-Agent header for all requests. // This helps identify your application in server logs and metrics. // // Parameters: // - appName: Name of the application // - appVersion: Version of the application // // Both parameters must be non-empty for the custom User-Agent to be set. func (c *Client) SetAppInfo(appName, appVersion string) { // if app name and version not set, we do not set a new user agent. if appName != "" && appVersion != "" { c.appInfo.appName = appName c.appInfo.appVersion = appVersion } } // TraceOn enables HTTP request and response tracing for debugging purposes. // All HTTP traffic will be written to the provided output stream. // // Parameters: // - outputStream: Writer where trace output will be written (defaults to os.Stdout if nil) func (c *Client) TraceOn(outputStream io.Writer) { // if outputStream is nil then default to os.Stdout. if outputStream == nil { outputStream = os.Stdout } // Sets a new output stream. c.traceOutput = outputStream // Enable tracing. c.isTraceEnabled = true } // TraceErrorsOnlyOn enables HTTP tracing but only for requests that result in errors. // This is useful for debugging without the overhead of tracing all requests. // // Parameters: // - outputStream: Writer where trace output will be written (defaults to os.Stdout if nil) func (c *Client) TraceErrorsOnlyOn(outputStream io.Writer) { c.TraceOn(outputStream) c.traceErrorsOnly = true } // TraceErrorsOnlyOff disables errors-only mode and traces all requests. // To disable all tracing, call TraceOff() instead. func (c *Client) TraceErrorsOnlyOff() { c.traceErrorsOnly = false } // TraceOff disables all HTTP tracing (both normal and errors-only modes). func (c *Client) TraceOff() { // Disable tracing. c.isTraceEnabled = false c.traceErrorsOnly = false } // SetS3TransferAccelerate - turns s3 accelerated endpoint on or off for all your // requests. This feature is only specific to S3 for all other endpoints this // function does nothing. To read further details on s3 transfer acceleration // please vist - // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) { if s3utils.IsAmazonEndpoint(*c.endpointURL) { c.s3AccelerateEndpoint = accelerateEndpoint } } // SetS3EnableDualstack turns s3 dual-stack endpoints on or off for all requests. // The feature is only specific to S3 and is on by default. To read more about // Amazon S3 dual-stack endpoints visit - // https://docs.aws.amazon.com/AmazonS3/latest/userguide/dual-stack-endpoints.html func (c *Client) SetS3EnableDualstack(enabled bool) { if s3utils.IsAmazonEndpoint(*c.endpointURL) { c.s3DualstackEnabled = enabled } } // Hash materials provides relevant initialized hash algo writers // based on the expected signature type. // // - For signature v4 request if the connection is insecure compute only sha256. // - For signature v4 request if the connection is secure compute only md5. // - For anonymous request compute md5. func (c *Client) hashMaterials(isMd5Requested, isSha256Requested bool) (hashAlgos map[string]md5simd.Hasher, hashSums map[string][]byte) { hashSums = make(map[string][]byte) hashAlgos = make(map[string]md5simd.Hasher) if c.overrideSignerType.IsV4() { if c.secure { hashAlgos["md5"] = c.md5Hasher() } else { if isSha256Requested { hashAlgos["sha256"] = c.sha256Hasher() } } } else { if c.overrideSignerType.IsAnonymous() { hashAlgos["md5"] = c.md5Hasher() } } if isMd5Requested { hashAlgos["md5"] = c.md5Hasher() } return hashAlgos, hashSums } const ( unknown = -1 offline = 0 online = 1 ) // IsOnline returns true if healthcheck enabled and client is online. // If HealthCheck function has not been called this will always return true. func (c *Client) IsOnline() bool { return !c.IsOffline() } // sets online healthStatus to offline func (c *Client) markOffline() { atomic.CompareAndSwapInt32(&c.healthStatus, online, offline) } // IsOffline returns true if healthcheck enabled and client is offline // If HealthCheck function has not been called this will always return false. func (c *Client) IsOffline() bool { return atomic.LoadInt32(&c.healthStatus) == offline } // HealthCheck starts a healthcheck to see if endpoint is up. // Returns a context cancellation function, to stop the health check, // and an error if health check is already started. func (c *Client) HealthCheck(hcDuration time.Duration) (context.CancelFunc, error) { if atomic.LoadInt32(&c.healthStatus) != unknown { return nil, fmt.Errorf("health check is running") } if hcDuration < 1*time.Second { return nil, fmt.Errorf("health check duration should be at least 1 second") } probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-health-") ctx, cancelFn := context.WithCancel(context.Background()) atomic.StoreInt32(&c.healthStatus, offline) { // Change to online, if we can connect. gctx, gcancel := context.WithTimeout(ctx, 3*time.Second) _, err := c.getBucketLocation(gctx, probeBucketName) gcancel() if !IsNetworkOrHostDown(err, false) { switch ToErrorResponse(err).Code { case NoSuchBucket, AccessDenied, "": atomic.CompareAndSwapInt32(&c.healthStatus, offline, online) } } } go func(duration time.Duration) { timer := time.NewTimer(duration) defer timer.Stop() for { select { case <-ctx.Done(): atomic.StoreInt32(&c.healthStatus, unknown) return case <-timer.C: // Do health check the first time and ONLY if the connection is marked offline if c.IsOffline() { gctx, gcancel := context.WithTimeout(context.Background(), 3*time.Second) _, err := c.getBucketLocation(gctx, probeBucketName) gcancel() if !IsNetworkOrHostDown(err, false) { switch ToErrorResponse(err).Code { case NoSuchBucket, AccessDenied, "": atomic.CompareAndSwapInt32(&c.healthStatus, offline, online) } } } timer.Reset(duration) } } }(hcDuration) return cancelFn, nil } // requestMetadata - is container for all the values to make a request. type requestMetadata struct { // If set newRequest presigns the URL. presignURL bool // User supplied. bucketName string objectName string queryValues url.Values customHeader http.Header extraPresignHeader http.Header expires int64 // Generated by our internal code. bucketLocation string contentBody io.Reader contentLength int64 contentMD5Base64 string // carries base64 encoded md5sum contentSHA256Hex string // carries hex encoded sha256sum streamSha256 bool addCrc *ChecksumType trailer http.Header // (http.Request).Trailer. Requires v4 signature. expect200OKWithError bool } // dumpHTTP - dump HTTP request and response. func (c *Client) dumpHTTP(req *http.Request, resp *http.Response) error { // Starts http dump. _, err := fmt.Fprintln(c.traceOutput, "---------START-HTTP---------") if err != nil { return err } // Filter out Signature field from Authorization header. origAuth := req.Header.Get("Authorization") if origAuth != "" { req.Header.Set("Authorization", redactSignature(origAuth)) } // Only display request header. reqTrace, err := httputil.DumpRequestOut(req, false) if err != nil { return err } // Write request to trace output. _, err = fmt.Fprint(c.traceOutput, string(reqTrace)) if err != nil { return err } // Only display response header. var respTrace []byte // For errors we make sure to dump response body as well. if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusNoContent { respTrace, err = httputil.DumpResponse(resp, true) if err != nil { return err } } else { respTrace, err = httputil.DumpResponse(resp, false) if err != nil { return err } } // Write response to trace output. _, err = fmt.Fprint(c.traceOutput, strings.TrimSuffix(string(respTrace), "\r\n")) if err != nil { return err } // Ends the http dump. _, err = fmt.Fprintln(c.traceOutput, "---------END-HTTP---------") if err != nil { return err } // Returns success. return nil } // do - execute http request. func (c *Client) do(req *http.Request) (resp *http.Response, err error) { defer func() { if IsNetworkOrHostDown(err, false) { c.markOffline() } }() resp, err = c.httpClient.Do(req) if err != nil { // Handle this specifically for now until future Golang versions fix this issue properly. if urlErr, ok := err.(*url.Error); ok { if strings.Contains(urlErr.Err.Error(), "EOF") { return nil, &url.Error{ Op: urlErr.Op, URL: urlErr.URL, Err: errors.New("Connection closed by foreign host " + urlErr.URL + ". Retry again."), } } } return nil, err } // Response cannot be non-nil, report error if thats the case. if resp == nil { msg := "Response is empty. " + reportIssue return nil, errInvalidArgument(msg) } // If trace is enabled, dump http request and response, // except when the traceErrorsOnly enabled and the response's status code is ok if c.isTraceEnabled && (!c.traceErrorsOnly || resp.StatusCode != http.StatusOK) { err = c.dumpHTTP(req, resp) if err != nil { return nil, err } } return resp, nil } // List of success status. var successStatus = map[int]struct{}{ http.StatusOK: {}, http.StatusNoContent: {}, http.StatusPartialContent: {}, } // executeMethod - instantiates a given method, and retries the // request upon any error up to maxRetries attempts in a binomially // delayed manner using a standard back off algorithm. func (c *Client) executeMethod(ctx context.Context, method string, metadata requestMetadata) (res *http.Response, err error) { if c.IsOffline() { return nil, errors.New(c.endpointURL.String() + " is offline.") } var retryable bool // Indicates if request can be retried. var bodySeeker io.Seeker // Extracted seeker from io.Reader. reqRetry := c.maxRetries // Indicates how many times we can retry the request if metadata.contentBody != nil { // Check if body is seekable then it is retryable. bodySeeker, retryable = metadata.contentBody.(io.Seeker) switch bodySeeker { case os.Stdin, os.Stdout, os.Stderr: retryable = false } // Retry only when reader is seekable if !retryable { reqRetry = 1 } // Figure out if the body can be closed - if yes // we will definitely close it upon the function // return. bodyCloser, ok := metadata.contentBody.(io.Closer) if ok { defer bodyCloser.Close() } } if metadata.addCrc != nil && metadata.contentLength > 0 { if metadata.trailer == nil { metadata.trailer = make(http.Header, 1) } crc := metadata.addCrc.Hasher() metadata.contentBody = newHashReaderWrapper(metadata.contentBody, crc, func(hash []byte) { // Update trailer when done. metadata.trailer.Set(metadata.addCrc.Key(), base64.StdEncoding.EncodeToString(hash)) }) metadata.trailer.Set(metadata.addCrc.Key(), base64.StdEncoding.EncodeToString(crc.Sum(nil))) } for range c.newRetryTimer(ctx, reqRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter) { // Retry executes the following function body if request has an // error until maxRetries have been exhausted, retry attempts are // performed after waiting for a given period of time in a // binomial fashion. if retryable { // Seek back to beginning for each attempt. if _, err = bodySeeker.Seek(0, 0); err != nil { // If seek failed, no need to retry. return nil, err } } // Instantiate a new request. var req *http.Request req, err = c.newRequest(ctx, method, metadata) if err != nil { errResponse := ToErrorResponse(err) if isS3CodeRetryable(errResponse.Code) { continue // Retry. } return nil, err } // Initiate the request. res, err = c.do(req) if err != nil { if isRequestErrorRetryable(ctx, err) { // Retry the request continue } return nil, err } _, success := successStatus[res.StatusCode] if success && !metadata.expect200OKWithError { // We do not expect 2xx to return an error return. return res, nil } // in all other situations we must first parse the body as ErrorResponse // 5MiB is sufficiently large enough to hold any error or regular XML response. var bodyBytes []byte bodyBytes, err = io.ReadAll(io.LimitReader(res.Body, 5*humanize.MiByte)) // By now, res.Body should be closed closeResponse(res) if err != nil { return nil, err } // Save the body. bodySeeker := bytes.NewReader(bodyBytes) res.Body = io.NopCloser(bodySeeker) apiErr := httpRespToErrorResponse(res, metadata.bucketName, metadata.objectName) // Save the body back again. bodySeeker.Seek(0, 0) // Seek back to starting point. res.Body = io.NopCloser(bodySeeker) if apiErr == nil { return res, nil } // For errors verify if its retryable otherwise fail quickly. errResponse := ToErrorResponse(apiErr) err = errResponse // Bucket region if set in error response and the error // code dictates invalid region, we can retry the request // with the new region. // // Additionally, we should only retry if bucketLocation and custom // region is empty. if c.region == "" { switch errResponse.Code { case AuthorizationHeaderMalformed: fallthrough case InvalidRegion: fallthrough case AccessDenied: if errResponse.Region == "" { // Region is empty we simply return the error. return res, err } // Region is not empty figure out a way to // handle this appropriately. if metadata.bucketName != "" { // Gather Cached location only if bucketName is present. if location, cachedOk := c.bucketLocCache.Get(metadata.bucketName); cachedOk && location != errResponse.Region { c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) continue // Retry. } } else { // This is for ListBuckets() fallback. if errResponse.Region != metadata.bucketLocation { // Retry if the error response has a different region // than the request we just made. metadata.bucketLocation = errResponse.Region continue // Retry } } } } // Verify if error response code is retryable. if isS3CodeRetryable(errResponse.Code) { continue // Retry. } // Verify if http status code is retryable. if isHTTPStatusRetryable(res.StatusCode) { continue // Retry. } // For all other cases break out of the retry loop. break } // Return an error when retry is canceled or deadlined if e := ctx.Err(); e != nil { return nil, e } return res, err } // newRequest - instantiate a new HTTP request for a given method. func (c *Client) newRequest(ctx context.Context, method string, metadata requestMetadata) (req *http.Request, err error) { // If no method is supplied default to 'POST'. if method == "" { method = http.MethodPost } location := metadata.bucketLocation if location == "" { if metadata.bucketName != "" { // Gather location only if bucketName is present. location, err = c.getBucketLocation(ctx, metadata.bucketName) if err != nil { return nil, err } } if location == "" { location = getDefaultLocation(*c.endpointURL, c.region) } } // Look if target url supports virtual host. // We explicitly disallow MakeBucket calls to not use virtual DNS style, // since the resolution may fail. isMakeBucket := (metadata.objectName == "" && method == http.MethodPut && len(metadata.queryValues) == 0) isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, metadata.bucketName) && !isMakeBucket // Construct a new target URL. targetURL, err := c.makeTargetURL(metadata.bucketName, metadata.objectName, location, isVirtualHost, metadata.queryValues) if err != nil { return nil, err } if c.httpTrace != nil { ctx = httptrace.WithClientTrace(ctx, c.httpTrace) } // make sure to de-dup calls to credential services, this reduces // the overall load to the endpoint generating credential service. value, err, _ := c.credsGroup.Do(metadata.bucketName, func() (credentials.Value, error) { if s3utils.IsS3ExpressBucket(metadata.bucketName) && s3utils.IsAmazonEndpoint(*c.endpointURL) { return c.CreateSession(ctx, metadata.bucketName, SessionReadWrite) } // Get credentials from the configured credentials provider. return c.credsProvider.GetWithContext(c.CredContext()) }) if err != nil { return nil, err } // Initialize a new HTTP request for the method. req, err = http.NewRequestWithContext(ctx, method, targetURL.String(), nil) if err != nil { return nil, err } var ( signerType = value.SignerType accessKeyID = value.AccessKeyID secretAccessKey = value.SecretAccessKey sessionToken = value.SessionToken ) if s3utils.IsS3ExpressBucket(metadata.bucketName) && sessionToken != "" { req.Header.Set("x-amz-s3session-token", sessionToken) } // Custom signer set then override the behavior. if c.overrideSignerType != credentials.SignatureDefault { signerType = c.overrideSignerType } // If signerType returned by credentials helper is anonymous, // then do not sign regardless of signerType override. if value.SignerType == credentials.SignatureAnonymous { signerType = credentials.SignatureAnonymous } // Generate presign url if needed, return right here. if metadata.expires != 0 && metadata.presignURL { if signerType.IsAnonymous() { return nil, errInvalidArgument("Presigned URLs cannot be generated with anonymous credentials.") } if metadata.extraPresignHeader != nil { if signerType.IsV2() { return nil, errInvalidArgument("Extra signed headers for Presign with Signature V2 is not supported.") } for k, v := range metadata.extraPresignHeader { req.Header.Set(k, v[0]) } } if signerType.IsV2() { // Presign URL with signature v2. req = signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost) } else if signerType.IsV4() { // Presign URL with signature v4. req = signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires) } return req, nil } // Set 'User-Agent' header for the request. c.setUserAgent(req) // Set all headers. for k, v := range metadata.customHeader { req.Header.Set(k, v[0]) } // Go net/http notoriously closes the request body. // - The request Body, if non-nil, will be closed by the underlying Transport, even on errors. // This can cause underlying *os.File seekers to fail, avoid that // by making sure to wrap the closer as a nop. if metadata.contentLength == 0 { req.Body = nil } else { req.Body = io.NopCloser(metadata.contentBody) } // Set incoming content-length. req.ContentLength = metadata.contentLength if req.ContentLength <= -1 { // For unknown content length, we upload using transfer-encoding: chunked. req.TransferEncoding = []string{"chunked"} } // set md5Sum for content protection. if len(metadata.contentMD5Base64) > 0 { req.Header.Set("Content-Md5", metadata.contentMD5Base64) } // For anonymous requests just return. if signerType.IsAnonymous() { if len(metadata.trailer) > 0 { req.Header.Set("X-Amz-Content-Sha256", unsignedPayloadTrailer) return signer.UnsignedTrailer(*req, metadata.trailer), nil } return req, nil } switch { case signerType.IsV2(): // Add signature version '2' authorization header. req = signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualHost) case metadata.streamSha256 && !c.secure: if len(metadata.trailer) > 0 { req.Trailer = metadata.trailer } // Streaming signature is used by default for a PUT object request. // Additionally, we also look if the initialized client is secure, // if yes then we don't need to perform streaming signature. if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) { req = signer.StreamingSignV4Express(req, accessKeyID, secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher()) } else { req = signer.StreamingSignV4(req, accessKeyID, secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher()) } default: // Set sha256 sum for signature calculation only with signature version '4'. shaHeader := unsignedPayload if metadata.contentSHA256Hex != "" { shaHeader = metadata.contentSHA256Hex if len(metadata.trailer) > 0 { // Sanity check, we should not end up here if upstream is sane. return nil, errors.New("internal error: contentSHA256Hex with trailer not supported") } } else if len(metadata.trailer) > 0 { shaHeader = unsignedPayloadTrailer } req.Header.Set("X-Amz-Content-Sha256", shaHeader) if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) { req = signer.SignV4TrailerExpress(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer) } else { // Add signature version '4' authorization header. req = signer.SignV4Trailer(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer) } } // Return request. return req, nil } // set User agent. func (c *Client) setUserAgent(req *http.Request) { req.Header.Set("User-Agent", libraryUserAgent) if c.appInfo.appName != "" && c.appInfo.appVersion != "" { req.Header.Set("User-Agent", libraryUserAgent+" "+c.appInfo.appName+"/"+c.appInfo.appVersion) } } // makeTargetURL make a new target url. func (c *Client) makeTargetURL(bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error) { host := c.endpointURL.Host // For Amazon S3 endpoint, try to fetch location based endpoint. if s3utils.IsAmazonEndpoint(*c.endpointURL) { if c.s3AccelerateEndpoint != "" && bucketName != "" { // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html // Disable transfer acceleration for non-compliant bucket names. if strings.Contains(bucketName, ".") { return nil, errTransferAccelerationBucket(bucketName) } // If transfer acceleration is requested set new host. // For more details about enabling transfer acceleration read here. // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html host = c.s3AccelerateEndpoint } else { // Do not change the host if the endpoint URL is a FIPS S3 endpoint or a S3 PrivateLink interface endpoint if !s3utils.IsAmazonFIPSEndpoint(*c.endpointURL) && !s3utils.IsAmazonPrivateLinkEndpoint(*c.endpointURL) { if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) { if bucketName == "" { host = getS3ExpressEndpoint(bucketLocation, false) } else { // Fetch new host based on the bucket location. host = getS3ExpressEndpoint(bucketLocation, s3utils.IsS3ExpressBucket(bucketName)) } } else { // Fetch new host based on the bucket location. host = getS3Endpoint(bucketLocation, c.s3DualstackEnabled) } } } } // Save scheme. scheme := c.endpointURL.Scheme // Strip port 80 and 443 so we won't send these ports in Host header. // The reason is that browsers and curl automatically remove :80 and :443 // with the generated presigned urls, then a signature mismatch error. if h, p, err := net.SplitHostPort(host); err == nil { if scheme == "http" && p == "80" || scheme == "https" && p == "443" { host = h if ip := net.ParseIP(h); ip != nil && ip.To4() == nil { host = "[" + h + "]" } } } urlStr := scheme + "://" + host + "/" // Make URL only if bucketName is available, otherwise use the // endpoint URL. if bucketName != "" { // If endpoint supports virtual host style use that always. // Currently only S3 and Google Cloud Storage would support // virtual host style. if isVirtualHostStyle { urlStr = scheme + "://" + bucketName + "." + host + "/" if objectName != "" { urlStr += s3utils.EncodePath(objectName) } } else { // If not fall back to using path style. urlStr = urlStr + bucketName + "/" if objectName != "" { urlStr += s3utils.EncodePath(objectName) } } } // If there are any query values, add them to the end. if len(queryValues) > 0 { urlStr = urlStr + "?" + s3utils.QueryEncode(queryValues) } return url.Parse(urlStr) } // returns true if virtual hosted style requests are to be used. func (c *Client) isVirtualHostStyleRequest(url url.URL, bucketName string) bool { if c.lookupFn != nil { lookup := c.lookupFn(url, bucketName) switch lookup { case BucketLookupDNS: return true case BucketLookupPath: return false } // if its auto then we fallback to default detection. return s3utils.IsVirtualHostSupported(url, bucketName) } if bucketName == "" { return false } if c.lookup == BucketLookupDNS { return true } if c.lookup == BucketLookupPath { return false } // default to virtual only for Amazon/Google storage. In all other cases use // path style requests return s3utils.IsVirtualHostSupported(url, bucketName) } // CredContext returns the context for fetching credentials func (c *Client) CredContext() *credentials.CredContext { httpClient := c.httpClient if httpClient == nil { httpClient = http.DefaultClient } return &credentials.CredContext{ Client: httpClient, Endpoint: c.endpointURL.String(), } } // GetCreds returns the access creds for the client func (c *Client) GetCreds() (credentials.Value, error) { if c.credsProvider == nil { return credentials.Value{}, errors.New("no credentials provider") } return c.credsProvider.GetWithContext(c.CredContext()) } minio-go-7.0.97/api_unit_test.go000066400000000000000000000202771510244170000165340ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2024 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "net/url" "testing" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/policy" ) // Tests valid hosts for location. func TestValidBucketLocation(t *testing.T) { s3Hosts := []struct { bucketLocation string useDualstack bool endpoint string }{ {"us-east-1", true, "s3.dualstack.us-east-1.amazonaws.com"}, {"us-east-1", false, "s3.us-east-1.amazonaws.com"}, {"unknown", true, "s3.dualstack.us-east-1.amazonaws.com"}, {"unknown", false, "s3.us-east-1.amazonaws.com"}, {"ap-southeast-1", true, "s3.dualstack.ap-southeast-1.amazonaws.com"}, {"ap-southeast-1", false, "s3.ap-southeast-1.amazonaws.com"}, } for _, s3Host := range s3Hosts { endpoint := getS3Endpoint(s3Host.bucketLocation, s3Host.useDualstack) if endpoint != s3Host.endpoint { t.Fatal("Error: invalid bucket location", endpoint) } } } // Tests error response structure. func TestErrorResponse(t *testing.T) { var err error err = ErrorResponse{ Code: Testing, } errResp := ToErrorResponse(err) if errResp.Code != Testing { t.Fatal("Type conversion failed, we have an empty struct.") } // Should fail with invalid argument. err = httpRespToErrorResponse(nil, "", "") errResp = ToErrorResponse(err) if errResp.Code != InvalidArgument { t.Fatal("Empty response input should return invalid argument.") } } // Tests signature type. func TestSignatureType(t *testing.T) { clnt := Client{} if !clnt.overrideSignerType.IsV4() { t.Fatal("Error") } clnt.overrideSignerType = credentials.SignatureV2 if !clnt.overrideSignerType.IsV2() { t.Fatal("Error") } if clnt.overrideSignerType.IsV4() { t.Fatal("Error") } clnt.overrideSignerType = credentials.SignatureV4 if !clnt.overrideSignerType.IsV4() { t.Fatal("Error") } } // Tests bucket policy types. func TestBucketPolicyTypes(t *testing.T) { want := map[string]bool{ "none": true, "readonly": true, "writeonly": true, "readwrite": true, "invalid": false, } for bucketPolicy, ok := range want { if policy.BucketPolicy(bucketPolicy).IsValidBucketPolicy() != ok { t.Fatal("Error") } } } // Tests optimal part size. func TestPartSize(t *testing.T) { _, _, _, err := OptimalPartInfo(5000000000000000000, minPartSize) if err == nil { t.Fatal("Error: should fail") } totalPartsCount, partSize, lastPartSize, err := OptimalPartInfo(5243928576, 5*1024*1024) if err != nil { t.Fatal("Error: ", err) } if totalPartsCount != 1001 { t.Fatalf("Error: expecting total parts count of 1001: got %v instead", totalPartsCount) } if partSize != 5242880 { t.Fatalf("Error: expecting part size of 5242880: got %v instead", partSize) } if lastPartSize != 1048576 { t.Fatalf("Error: expecting last part size of 1048576: got %v instead", lastPartSize) } totalPartsCount, partSize, lastPartSize, err = OptimalPartInfo(5243928576, 0) if err != nil { t.Fatal("Error: ", err) } if totalPartsCount != 313 { t.Fatalf("Error: expecting total parts count of 313: got %v instead", totalPartsCount) } if partSize != 16777216 { t.Fatalf("Error: expecting part size of 16777216: got %v instead", partSize) } if lastPartSize != 9437184 { t.Fatalf("Error: expecting last part size of 9437184: got %v instead", lastPartSize) } _, partSize, _, err = OptimalPartInfo(5000000000, minPartSize) if err != nil { t.Fatal("Error:", err) } if partSize != minPartSize { t.Fatalf("Error: expecting part size of %v: got %v instead", minPartSize, partSize) } // if stream and using default optimal part size determined by sdk totalPartsCount, partSize, lastPartSize, err = OptimalPartInfo(-1, 0) if err != nil { t.Fatal("Error:", err) } if totalPartsCount != 9930 { t.Fatalf("Error: expecting total parts count of 9930: got %v instead", totalPartsCount) } if partSize != 553648128 { t.Fatalf("Error: expecting part size of 553648128: got %v instead", partSize) } if lastPartSize != 385875968 { t.Fatalf("Error: expecting last part size of 385875968: got %v instead", lastPartSize) } totalPartsCount, partSize, lastPartSize, err = OptimalPartInfo(-1, 64*1024*1024) if err != nil { t.Fatal("Error:", err) } if totalPartsCount != 10000 { t.Fatalf("Error: expecting total parts count of 10000: got %v instead", totalPartsCount) } if partSize != 67108864 { t.Fatalf("Error: expecting part size of 67108864: got %v instead", partSize) } if lastPartSize != 67108864 { t.Fatalf("Error: expecting part size of 67108864: got %v instead", lastPartSize) } } // TestMakeTargetURL - testing makeTargetURL() func TestMakeTargetURL(t *testing.T) { testCases := []struct { addr string secure bool bucketName string objectName string bucketLocation string queryValues map[string][]string expectedURL url.URL expectedErr error }{ // Test 1 {"localhost:9000", false, "", "", "", nil, url.URL{Host: "localhost:9000", Scheme: "http", Path: "/"}, nil}, // Test 2 {"localhost", true, "", "", "", nil, url.URL{Host: "localhost", Scheme: "https", Path: "/"}, nil}, // Test 3 {"localhost:9000", true, "mybucket", "", "", nil, url.URL{Host: "localhost:9000", Scheme: "https", Path: "/mybucket/"}, nil}, // Test 4, testing against google storage API {"storage.googleapis.com", true, "mybucket", "", "", nil, url.URL{Host: "mybucket.storage.googleapis.com", Scheme: "https", Path: "/"}, nil}, // Test 5, testing against AWS S3 API {"s3.amazonaws.com", true, "mybucket", "myobject", "", nil, url.URL{Host: "mybucket.s3.dualstack.us-east-1.amazonaws.com", Scheme: "https", Path: "/myobject"}, nil}, // Test 6 {"localhost:9000", false, "mybucket", "myobject", "", nil, url.URL{Host: "localhost:9000", Scheme: "http", Path: "/mybucket/myobject"}, nil}, // Test 7, testing with query {"localhost:9000", false, "mybucket", "myobject", "", map[string][]string{"param": {"val"}}, url.URL{Host: "localhost:9000", Scheme: "http", Path: "/mybucket/myobject", RawQuery: "param=val"}, nil}, // Test 8, testing with port 80 {"localhost:80", false, "mybucket", "myobject", "", nil, url.URL{Host: "localhost", Scheme: "http", Path: "/mybucket/myobject"}, nil}, // Test 9, testing with port 443 {"localhost:443", true, "mybucket", "myobject", "", nil, url.URL{Host: "localhost", Scheme: "https", Path: "/mybucket/myobject"}, nil}, {"[240b:c0e0:102:54C0:1c05:c2c1:19:5001]:443", true, "mybucket", "myobject", "", nil, url.URL{Host: "[240b:c0e0:102:54C0:1c05:c2c1:19:5001]", Scheme: "https", Path: "/mybucket/myobject"}, nil}, {"[240b:c0e0:102:54C0:1c05:c2c1:19:5001]:9000", true, "mybucket", "myobject", "", nil, url.URL{Host: "[240b:c0e0:102:54C0:1c05:c2c1:19:5001]:9000", Scheme: "https", Path: "/mybucket/myobject"}, nil}, } for i, testCase := range testCases { // Initialize a MinIO client c, _ := New(testCase.addr, &Options{ Creds: credentials.NewStaticV4("foo", "bar", ""), Secure: testCase.secure, }) isVirtualHost := c.isVirtualHostStyleRequest(*c.endpointURL, testCase.bucketName) u, err := c.makeTargetURL(testCase.bucketName, testCase.objectName, testCase.bucketLocation, isVirtualHost, testCase.queryValues) // Check the returned error if testCase.expectedErr == nil && err != nil { t.Fatalf("Test %d: Should succeed but failed with err = %v", i+1, err) } if testCase.expectedErr != nil && err == nil { t.Fatalf("Test %d: Should fail but succeeded", i+1) } if err == nil { // Check if the returned url is equal to what we expect if u.String() != testCase.expectedURL.String() { t.Fatalf("Test %d: Mismatched target url: expected = `%v`, found = `%v`", i+1, testCase.expectedURL.String(), u.String()) } } } } minio-go-7.0.97/bucket-cache.go000066400000000000000000000135051510244170000161770ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "net" "net/http" "net/url" "path" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/s3utils" "github.com/minio/minio-go/v7/pkg/signer" ) // GetBucketLocation - get location for the bucket name from location cache, if not // fetch freshly by making a new request. func (c *Client) GetBucketLocation(ctx context.Context, bucketName string) (string, error) { if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", err } return c.getBucketLocation(ctx, bucketName) } // getBucketLocation - Get location for the bucketName from location map cache, if not // fetch freshly by making a new request. func (c *Client) getBucketLocation(ctx context.Context, bucketName string) (string, error) { if err := s3utils.CheckValidBucketName(bucketName); err != nil { return "", err } // Region set then no need to fetch bucket location. if c.region != "" { return c.region, nil } if location, ok := c.bucketLocCache.Get(bucketName); ok { return location, nil } // Initialize a new request. req, err := c.getBucketLocationRequest(ctx, bucketName) if err != nil { return "", err } // Initiate the request. resp, err := c.do(req) defer closeResponse(resp) if err != nil { return "", err } location, err := processBucketLocationResponse(resp, bucketName) if err != nil { return "", err } c.bucketLocCache.Set(bucketName, location) return location, nil } // processes the getBucketLocation http response from the server. func processBucketLocationResponse(resp *http.Response, bucketName string) (bucketLocation string, err error) { if resp != nil { if resp.StatusCode != http.StatusOK { err = httpRespToErrorResponse(resp, bucketName, "") errResp := ToErrorResponse(err) // For access denied error, it could be an anonymous // request. Move forward and let the top level callers // succeed if possible based on their policy. switch errResp.Code { case NotImplemented: switch errResp.Server { case "AmazonSnowball": return "snowball", nil case "cloudflare": return "us-east-1", nil } case AuthorizationHeaderMalformed: fallthrough case InvalidRegion: fallthrough case AccessDenied: if errResp.Region == "" { return "us-east-1", nil } return errResp.Region, nil } return "", err } } // Extract location. var locationConstraint string err = xmlDecoder(resp.Body, &locationConstraint) if err != nil { return "", err } location := locationConstraint // Location is empty will be 'us-east-1'. if location == "" { location = "us-east-1" } // Location can be 'EU' convert it to meaningful 'eu-west-1'. if location == "EU" { location = "eu-west-1" } // Save the location into cache. // Return. return location, nil } // getBucketLocationRequest - Wrapper creates a new getBucketLocation request. func (c *Client) getBucketLocationRequest(ctx context.Context, bucketName string) (*http.Request, error) { // Set location query. urlValues := make(url.Values) urlValues.Set("location", "") // Set get bucket location always as path style. targetURL := *c.endpointURL // as it works in makeTargetURL method from api.go file if h, p, err := net.SplitHostPort(targetURL.Host); err == nil { if targetURL.Scheme == "http" && p == "80" || targetURL.Scheme == "https" && p == "443" { targetURL.Host = h if ip := net.ParseIP(h); ip != nil && ip.To4() == nil { targetURL.Host = "[" + h + "]" } } } isVirtualStyle := c.isVirtualHostStyleRequest(targetURL, bucketName) var urlStr string if isVirtualStyle { urlStr = c.endpointURL.Scheme + "://" + bucketName + "." + targetURL.Host + "/?location" } else { targetURL.Path = path.Join(bucketName, "") + "/" targetURL.RawQuery = urlValues.Encode() urlStr = targetURL.String() } // Get a new HTTP request for the method. req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil) if err != nil { return nil, err } // Set UserAgent for the request. c.setUserAgent(req) // Get credentials from the configured credentials provider. value, err := c.credsProvider.GetWithContext(c.CredContext()) if err != nil { return nil, err } var ( signerType = value.SignerType accessKeyID = value.AccessKeyID secretAccessKey = value.SecretAccessKey sessionToken = value.SessionToken ) // Custom signer set then override the behavior. if c.overrideSignerType != credentials.SignatureDefault { signerType = c.overrideSignerType } // If signerType returned by credentials helper is anonymous, // then do not sign regardless of signerType override. if value.SignerType == credentials.SignatureAnonymous { signerType = credentials.SignatureAnonymous } if signerType.IsAnonymous() { return req, nil } if signerType.IsV2() { req = signer.SignV2(*req, accessKeyID, secretAccessKey, isVirtualStyle) return req, nil } // Set sha256 sum for signature calculation only with signature version '4'. contentSha256 := emptySHA256Hex if c.secure { contentSha256 = unsignedPayload } req.Header.Set("X-Amz-Content-Sha256", contentSha256) req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") return req, nil } minio-go-7.0.97/bucket-cache_test.go000066400000000000000000000274411510244170000172420ustar00rootroot00000000000000/* * Copyright * 2015, 2016, 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "encoding/xml" "io" "net/http" "net/url" "path" "reflect" "testing" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/kvcache" "github.com/minio/minio-go/v7/pkg/signer" ) // Tests validate kvCache operations. func TestBucketLocationCacheOps(t *testing.T) { testBucketLocationCache := &kvcache.Cache[string, string]{} expectedBucketName := "minio-bucket" expectedLocation := "us-east-1" testBucketLocationCache.Set(expectedBucketName, expectedLocation) actualLocation, ok := testBucketLocationCache.Get(expectedBucketName) if !ok { t.Errorf("Bucket location cache not set") } if expectedLocation != actualLocation { t.Errorf("Bucket location cache not set to expected value") } testBucketLocationCache.Delete(expectedBucketName) _, ok = testBucketLocationCache.Get(expectedBucketName) if ok { t.Errorf("Bucket location cache not deleted as expected") } } // Tests validate http request generation for 'getBucketLocation'. func TestGetBucketLocationRequest(t *testing.T) { // Generates expected http request for getBucketLocation. // Used for asserting with the actual request generated. createExpectedRequest := func(c *Client, bucketName string) (*http.Request, error) { // Set location query. urlValues := make(url.Values) urlValues.Set("location", "") // Set get bucket location always as path style. targetURL := *c.endpointURL isVirtualStyle := c.isVirtualHostStyleRequest(targetURL, bucketName) var urlStr string if isVirtualStyle { urlStr = targetURL.Scheme + "://" + bucketName + "." + targetURL.Host + "/?location" } else { targetURL.Path = path.Join(bucketName, "") + "/" targetURL.RawQuery = urlValues.Encode() urlStr = targetURL.String() } // Get a new HTTP request for the method. req, err := http.NewRequest(http.MethodGet, urlStr, nil) if err != nil { return nil, err } // Set UserAgent for the request. c.setUserAgent(req) // Get credentials from the configured credentials provider. value, err := c.credsProvider.GetWithContext(c.CredContext()) if err != nil { return nil, err } var ( signerType = value.SignerType accessKeyID = value.AccessKeyID secretAccessKey = value.SecretAccessKey sessionToken = value.SessionToken ) // Custom signer set then override the behavior. if c.overrideSignerType != credentials.SignatureDefault { signerType = c.overrideSignerType } // If signerType returned by credentials helper is anonymous, // then do not sign regardless of signerType override. if value.SignerType == credentials.SignatureAnonymous { signerType = credentials.SignatureAnonymous } // Set sha256 sum for signature calculation only // with signature version '4'. switch { case signerType.IsV4(): contentSha256 := emptySHA256Hex if c.secure { contentSha256 = unsignedPayload } req.Header.Set("X-Amz-Content-Sha256", contentSha256) req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") case signerType.IsV2(): req = signer.SignV2(*req, accessKeyID, secretAccessKey, false) } return req, nil } // Info for 'Client' creation. // Will be used as arguments for 'NewClient'. type infoForClient struct { endPoint string accessKey string secretKey string enableInsecure bool } // dataset for 'NewClient' call. info := []infoForClient{ // endpoint localhost. // both access-key and secret-key are empty. {"localhost:9000", "", "", false}, // both access-key are secret-key exists. {"localhost:9000", "my-access-key", "my-secret-key", false}, // one of acess-key and secret-key are empty. {"localhost:9000", "", "my-secret-key", false}, // endpoint amazon s3. {"s3.amazonaws.com", "", "", false}, {"s3.amazonaws.com", "my-access-key", "my-secret-key", false}, {"s3.amazonaws.com", "my-acess-key", "", false}, // endpoint google cloud storage. {"storage.googleapis.com", "", "", false}, {"storage.googleapis.com", "my-access-key", "my-secret-key", false}, {"storage.googleapis.com", "", "my-secret-key", false}, // endpoint custom domain running MinIO server. {"play.min.io", "", "", false}, {"play.min.io", "my-access-key", "my-secret-key", false}, {"play.min.io", "my-acess-key", "", false}, } testCases := []struct { bucketName string // data for new client creation. info infoForClient // error in the output. err error // flag indicating whether tests should pass. shouldPass bool }{ // Client is constructed using the info struct. // case with empty location. {"my-bucket", info[0], nil, true}, // case with location set to standard 'us-east-1'. {"my-bucket", info[0], nil, true}, // case with location set to a value different from 'us-east-1'. {"my-bucket", info[0], nil, true}, {"my-bucket", info[1], nil, true}, {"my-bucket", info[1], nil, true}, {"my-bucket", info[1], nil, true}, {"my-bucket", info[2], nil, true}, {"my-bucket", info[2], nil, true}, {"my-bucket", info[2], nil, true}, {"my-bucket", info[3], nil, true}, {"my-bucket", info[3], nil, true}, {"my-bucket", info[3], nil, true}, {"my-bucket", info[4], nil, true}, {"my-bucket", info[4], nil, true}, {"my-bucket", info[4], nil, true}, {"my-bucket", info[5], nil, true}, {"my-bucket", info[5], nil, true}, {"my-bucket", info[5], nil, true}, {"my-bucket", info[6], nil, true}, {"my-bucket", info[6], nil, true}, {"my-bucket", info[6], nil, true}, {"my-bucket", info[7], nil, true}, {"my-bucket", info[7], nil, true}, {"my-bucket", info[7], nil, true}, {"my-bucket", info[8], nil, true}, {"my-bucket", info[8], nil, true}, {"my-bucket", info[8], nil, true}, {"my-bucket", info[9], nil, true}, {"my-bucket", info[9], nil, true}, {"my-bucket", info[9], nil, true}, {"my-bucket", info[10], nil, true}, {"my-bucket", info[10], nil, true}, {"my-bucket", info[10], nil, true}, {"my-bucket", info[11], nil, true}, {"my-bucket", info[11], nil, true}, {"my-bucket", info[11], nil, true}, } for i, testCase := range testCases { // cannot create a newclient with empty endPoint value. // validates and creates a new client only if the endPoint value is not empty. client := &Client{} var err error if testCase.info.endPoint != "" { client, err = New(testCase.info.endPoint, &Options{ Creds: credentials.NewStaticV4(testCase.info.accessKey, testCase.info.secretKey, ""), Secure: testCase.info.enableInsecure, }) if err != nil { t.Fatalf("Test %d: Failed to create new Client: %s", i+1, err.Error()) } } actualReq, err := client.getBucketLocationRequest(context.Background(), testCase.bucketName) if err != nil && testCase.shouldPass { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) } if err == nil && !testCase.shouldPass { t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) } // Failed as expected, but does it fail for the expected reason. if err != nil && !testCase.shouldPass { if err.Error() != testCase.err.Error() { t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) } } // Test passes as expected, but the output values are verified for correctness here. if err == nil && testCase.shouldPass { expectedReq, err := createExpectedRequest(client, testCase.bucketName) if err != nil { t.Fatalf("Test %d: Expected request Creation failed", i+1) } if expectedReq.Method != actualReq.Method { t.Errorf("Test %d: The expected Request method doesn't match with the actual one", i+1) } if expectedReq.URL.String() != actualReq.URL.String() { t.Errorf("Test %d: Expected the request URL to be '%s', but instead found '%s'", i+1, expectedReq.URL.String(), actualReq.URL.String()) } if expectedReq.ContentLength != actualReq.ContentLength { t.Errorf("Test %d: Expected the request body Content-Length to be '%d', but found '%d' instead", i+1, expectedReq.ContentLength, actualReq.ContentLength) } if expectedReq.Header.Get("X-Amz-Content-Sha256") != actualReq.Header.Get("X-Amz-Content-Sha256") { t.Errorf("Test %d: 'X-Amz-Content-Sha256' header of the expected request doesn't match with that of the actual request", i+1) } if expectedReq.Header.Get("User-Agent") != actualReq.Header.Get("User-Agent") { t.Errorf("Test %d: Expected 'User-Agent' header to be \"%s\",but found \"%s\" instead", i+1, expectedReq.Header.Get("User-Agent"), actualReq.Header.Get("User-Agent")) } } } } // generates http response with bucket location set in the body. func generateLocationResponse(resp *http.Response, bodyContent []byte) (*http.Response, error) { resp.StatusCode = http.StatusOK resp.Body = io.NopCloser(bytes.NewBuffer(bodyContent)) return resp, nil } // Tests the processing of GetPolicy response from server. func TestProcessBucketLocationResponse(t *testing.T) { // LocationResponse - format for location response. type LocationResponse struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"` Location string `xml:",chardata"` } APIErrors := []APIError{ { Code: AccessDenied, Description: "Access Denied", HTTPStatusCode: http.StatusUnauthorized, }, } testCases := []struct { bucketName string inputLocation string isAPIError bool apiErr APIError // expected results. expectedResult string err error // flag indicating whether tests should pass. shouldPass bool }{ {"my-bucket", "", true, APIErrors[0], "us-east-1", nil, true}, {"my-bucket", "", false, APIError{}, "us-east-1", nil, true}, {"my-bucket", "EU", false, APIError{}, "eu-west-1", nil, true}, {"my-bucket", "eu-central-1", false, APIError{}, "eu-central-1", nil, true}, {"my-bucket", "us-east-1", false, APIError{}, "us-east-1", nil, true}, } for i, testCase := range testCases { inputResponse := &http.Response{} var err error if testCase.isAPIError { inputResponse = generateErrorResponse(inputResponse, testCase.apiErr, testCase.bucketName) } else { inputResponse, err = generateLocationResponse(inputResponse, encodeResponse(LocationResponse{ Location: testCase.inputLocation, })) if err != nil { t.Fatalf("Test %d: Creation of valid response failed", i+1) } } actualResult, err := processBucketLocationResponse(inputResponse, "my-bucket") if err != nil && testCase.shouldPass { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) } if err == nil && !testCase.shouldPass { t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) } // Failed as expected, but does it fail for the expected reason. if err != nil && !testCase.shouldPass { if err.Error() != testCase.err.Error() { t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) } } if err == nil && testCase.shouldPass { if !reflect.DeepEqual(testCase.expectedResult, actualResult) { t.Errorf("Test %d: The expected BucketPolicy doesn't match the actual BucketPolicy", i+1) } } } } minio-go-7.0.97/checksum.go000066400000000000000000000300361510244170000154610ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2023 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "crypto/sha1" "crypto/sha256" "encoding/base64" "encoding/binary" "errors" "hash" "io" "math/bits" "net/http" "sort" "strings" "github.com/klauspost/crc32" "github.com/minio/crc64nvme" ) // ChecksumMode contains information about the checksum mode on the object type ChecksumMode uint32 const ( // ChecksumFullObjectMode Full object checksum `csumCombine(csum1, csum2...)...), csumN...)` ChecksumFullObjectMode ChecksumMode = 1 << iota // ChecksumCompositeMode Composite checksum `csum([csum1 + csum2 ... + csumN])` ChecksumCompositeMode // Keep after all valid checksums checksumLastMode // checksumModeMask is a mask for valid checksum mode types. checksumModeMask = checksumLastMode - 1 ) // Is returns if c is all of t. func (c ChecksumMode) Is(t ChecksumMode) bool { return c&t == t } // Key returns the header key. func (c ChecksumMode) Key() string { return amzChecksumMode } func (c ChecksumMode) String() string { switch c & checksumModeMask { case ChecksumFullObjectMode: return "FULL_OBJECT" case ChecksumCompositeMode: return "COMPOSITE" } return "" } // ChecksumType contains information about the checksum type. type ChecksumType uint32 const ( // ChecksumSHA256 indicates a SHA256 checksum. ChecksumSHA256 ChecksumType = 1 << iota // ChecksumSHA1 indicates a SHA-1 checksum. ChecksumSHA1 // ChecksumCRC32 indicates a CRC32 checksum with IEEE table. ChecksumCRC32 // ChecksumCRC32C indicates a CRC32 checksum with Castagnoli table. ChecksumCRC32C // ChecksumCRC64NVME indicates CRC64 with 0xad93d23594c93659 polynomial. ChecksumCRC64NVME // Keep after all valid checksums checksumLast // ChecksumFullObject is a modifier that can be used on CRC32 and CRC32C // to indicate full object checksums. ChecksumFullObject // checksumMask is a mask for valid checksum types. checksumMask = checksumLast - 1 // ChecksumNone indicates no checksum. ChecksumNone ChecksumType = 0 // ChecksumFullObjectCRC32 indicates full object CRC32 ChecksumFullObjectCRC32 = ChecksumCRC32 | ChecksumFullObject // ChecksumFullObjectCRC32C indicates full object CRC32C ChecksumFullObjectCRC32C = ChecksumCRC32C | ChecksumFullObject amzChecksumAlgo = "x-amz-checksum-algorithm" amzChecksumCRC32 = "x-amz-checksum-crc32" amzChecksumCRC32C = "x-amz-checksum-crc32c" amzChecksumSHA1 = "x-amz-checksum-sha1" amzChecksumSHA256 = "x-amz-checksum-sha256" amzChecksumCRC64NVME = "x-amz-checksum-crc64nvme" amzChecksumMode = "x-amz-checksum-type" ) // Base returns the base type, without modifiers. func (c ChecksumType) Base() ChecksumType { return c & checksumMask } // Is returns if c is all of t. func (c ChecksumType) Is(t ChecksumType) bool { return c&t == t } // Key returns the header key. // returns empty string if invalid or none. func (c ChecksumType) Key() string { switch c & checksumMask { case ChecksumCRC32: return amzChecksumCRC32 case ChecksumCRC32C: return amzChecksumCRC32C case ChecksumSHA1: return amzChecksumSHA1 case ChecksumSHA256: return amzChecksumSHA256 case ChecksumCRC64NVME: return amzChecksumCRC64NVME } return "" } // CanComposite will return if the checksum type can be used for composite multipart upload on AWS. func (c ChecksumType) CanComposite() bool { switch c & checksumMask { case ChecksumSHA256, ChecksumSHA1, ChecksumCRC32, ChecksumCRC32C: return true } return false } // CanMergeCRC will return if the checksum type can be used for multipart upload on AWS. func (c ChecksumType) CanMergeCRC() bool { switch c & checksumMask { case ChecksumCRC32, ChecksumCRC32C, ChecksumCRC64NVME: return true } return false } // FullObjectRequested will return if the checksum type indicates full object checksum was requested. func (c ChecksumType) FullObjectRequested() bool { switch c & (ChecksumFullObject | checksumMask) { case ChecksumFullObjectCRC32C, ChecksumFullObjectCRC32, ChecksumCRC64NVME: return true } return false } // KeyCapitalized returns the capitalized key as used in HTTP headers. func (c ChecksumType) KeyCapitalized() string { return http.CanonicalHeaderKey(c.Key()) } // RawByteLen returns the size of the un-encoded checksum. func (c ChecksumType) RawByteLen() int { switch c & checksumMask { case ChecksumCRC32, ChecksumCRC32C: return 4 case ChecksumSHA1: return sha1.Size case ChecksumSHA256: return sha256.Size case ChecksumCRC64NVME: return crc64nvme.Size } return 0 } const crc64NVMEPolynomial = 0xad93d23594c93659 // Hasher returns a hasher corresponding to the checksum type. // Returns nil if no checksum. func (c ChecksumType) Hasher() hash.Hash { switch c & checksumMask { case ChecksumCRC32: return crc32.NewIEEE() case ChecksumCRC32C: return crc32.New(crc32.MakeTable(crc32.Castagnoli)) case ChecksumSHA1: return sha1.New() case ChecksumSHA256: return sha256.New() case ChecksumCRC64NVME: return crc64nvme.New() } return nil } // IsSet returns whether the type is valid and known. func (c ChecksumType) IsSet() bool { return bits.OnesCount32(uint32(c&checksumMask)) == 1 } // SetDefault will set the checksum if not already set. func (c *ChecksumType) SetDefault(t ChecksumType) { if !c.IsSet() { *c = t } } // EncodeToString the encoded hash value of the content provided in b. func (c ChecksumType) EncodeToString(b []byte) string { if !c.IsSet() { return "" } h := c.Hasher() h.Write(b) return base64.StdEncoding.EncodeToString(h.Sum(nil)) } // String returns the type as a string. // CRC32, CRC32C, SHA1, and SHA256 for valid values. // Empty string for unset and "" if not valid. func (c ChecksumType) String() string { switch c & checksumMask { case ChecksumCRC32: return "CRC32" case ChecksumCRC32C: return "CRC32C" case ChecksumSHA1: return "SHA1" case ChecksumSHA256: return "SHA256" case ChecksumNone: return "" case ChecksumCRC64NVME: return "CRC64NVME" } return "" } // ChecksumReader reads all of r and returns a checksum of type c. // Returns any error that may have occurred while reading. func (c ChecksumType) ChecksumReader(r io.Reader) (Checksum, error) { h := c.Hasher() if h == nil { return Checksum{}, nil } _, err := io.Copy(h, r) if err != nil { return Checksum{}, err } return NewChecksum(c, h.Sum(nil)), nil } // ChecksumBytes returns a checksum of the content b with type c. func (c ChecksumType) ChecksumBytes(b []byte) Checksum { h := c.Hasher() if h == nil { return Checksum{} } n, err := h.Write(b) if err != nil || n != len(b) { // Shouldn't happen with these checksummers. return Checksum{} } return NewChecksum(c, h.Sum(nil)) } // Checksum is a type and encoded value. type Checksum struct { Type ChecksumType r []byte } // NewChecksum sets the checksum to the value of b, // which is the raw hash output. // If the length of c does not match t.RawByteLen, // a checksum with ChecksumNone is returned. func NewChecksum(t ChecksumType, b []byte) Checksum { if t.IsSet() && len(b) == t.RawByteLen() { return Checksum{Type: t, r: b} } return Checksum{} } // NewChecksumString sets the checksum to the value of s, // which is the base 64 encoded raw hash output. // If the length of c does not match t.RawByteLen, it is not added. func NewChecksumString(t ChecksumType, s string) Checksum { b, _ := base64.StdEncoding.DecodeString(s) if t.IsSet() && len(b) == t.RawByteLen() { return Checksum{Type: t, r: b} } return Checksum{} } // IsSet returns whether the checksum is valid and known. func (c Checksum) IsSet() bool { return c.Type.IsSet() && len(c.r) == c.Type.RawByteLen() } // Encoded returns the encoded value. // Returns the empty string if not set or valid. func (c Checksum) Encoded() string { if !c.IsSet() { return "" } return base64.StdEncoding.EncodeToString(c.r) } // Raw returns the raw checksum value if set. func (c Checksum) Raw() []byte { if !c.IsSet() { return nil } return c.r } // CompositeChecksum returns the composite checksum of all provided parts. func (c ChecksumType) CompositeChecksum(p []ObjectPart) (*Checksum, error) { if !c.CanComposite() { return nil, errors.New("cannot do composite checksum") } sort.Slice(p, func(i, j int) bool { return p[i].PartNumber < p[j].PartNumber }) c = c.Base() crcBytes := make([]byte, 0, len(p)*c.RawByteLen()) for _, part := range p { pCrc, err := part.ChecksumRaw(c) if err != nil { return nil, err } crcBytes = append(crcBytes, pCrc...) } h := c.Hasher() h.Write(crcBytes) return &Checksum{Type: c, r: h.Sum(nil)}, nil } // FullObjectChecksum will return the full object checksum from provided parts. func (c ChecksumType) FullObjectChecksum(p []ObjectPart) (*Checksum, error) { if !c.CanMergeCRC() { return nil, errors.New("cannot merge this checksum type") } c = c.Base() sort.Slice(p, func(i, j int) bool { return p[i].PartNumber < p[j].PartNumber }) switch len(p) { case 0: return nil, errors.New("no parts given") case 1: check, err := p[0].ChecksumRaw(c) if err != nil { return nil, err } return &Checksum{ Type: c, r: check, }, nil } var merged uint32 var merged64 uint64 first, err := p[0].ChecksumRaw(c) if err != nil { return nil, err } sz := p[0].Size switch c { case ChecksumCRC32, ChecksumCRC32C: merged = binary.BigEndian.Uint32(first) case ChecksumCRC64NVME: merged64 = binary.BigEndian.Uint64(first) } poly32 := uint32(crc32.IEEE) if c.Is(ChecksumCRC32C) { poly32 = crc32.Castagnoli } for _, part := range p[1:] { if part.Size == 0 { continue } sz += part.Size pCrc, err := part.ChecksumRaw(c) if err != nil { return nil, err } switch c { case ChecksumCRC32, ChecksumCRC32C: merged = crc32Combine(poly32, merged, binary.BigEndian.Uint32(pCrc), part.Size) case ChecksumCRC64NVME: merged64 = crc64Combine(bits.Reverse64(crc64NVMEPolynomial), merged64, binary.BigEndian.Uint64(pCrc), part.Size) } } var tmp [8]byte switch c { case ChecksumCRC32, ChecksumCRC32C: binary.BigEndian.PutUint32(tmp[:], merged) return &Checksum{ Type: c, r: tmp[:4], }, nil case ChecksumCRC64NVME: binary.BigEndian.PutUint64(tmp[:], merged64) return &Checksum{ Type: c, r: tmp[:8], }, nil default: return nil, errors.New("unknown checksum type") } } func addAutoChecksumHeaders(opts *PutObjectOptions) { if opts.UserMetadata == nil { opts.UserMetadata = make(map[string]string, 1) } addChecksum := true for k := range opts.UserMetadata { if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") { addChecksum = false } } if addChecksum && opts.AutoChecksum.IsSet() { opts.UserMetadata[amzChecksumAlgo] = opts.AutoChecksum.String() if opts.AutoChecksum.FullObjectRequested() { opts.UserMetadata[amzChecksumMode] = ChecksumFullObjectMode.String() } } } func applyAutoChecksum(opts *PutObjectOptions, allParts []ObjectPart) { if !opts.AutoChecksum.IsSet() { return } if opts.AutoChecksum.CanComposite() && !opts.AutoChecksum.Is(ChecksumFullObject) { // Add composite hash of hashes. crc, err := opts.AutoChecksum.CompositeChecksum(allParts) if err == nil { opts.UserMetadata = map[string]string{ opts.AutoChecksum.Key(): crc.Encoded(), amzChecksumMode: ChecksumCompositeMode.String(), } } } else if opts.AutoChecksum.CanMergeCRC() { crc, err := opts.AutoChecksum.FullObjectChecksum(allParts) if err == nil { opts.UserMetadata = map[string]string{ opts.AutoChecksum.Key(): crc.Encoded(), amzChecksumMode: ChecksumFullObjectMode.String(), } } } } minio-go-7.0.97/code_of_conduct.md000066400000000000000000000070401510244170000167660ustar00rootroot00000000000000Contributor Covenant Code of Conduct ==================================== Our Pledge ---------- In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. Our Standards ------------- Examples of behavior that contributes to creating a positive environment include: - Using welcoming and inclusive language - Being respectful of differing viewpoints and experiences - Gracefully accepting constructive criticism - Focusing on what is best for the community - Showing empathy towards other community members Examples of unacceptable behavior by participants include: - The use of sexualized language or imagery and unwelcome sexual attention or advances - Trolling, insulting/derogatory comments, and personal or political attacks - Public or private harassment - Publishing others' private information, such as a physical or electronic address, without explicit permission - Other conduct which could reasonably be considered inappropriate in a professional setting Our Responsibilities -------------------- Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior, in compliance with the licensing terms applying to the Project developments. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. However, these actions shall respect the licensing terms of the Project Developments that will always supersede such Code of Conduct. Scope ----- This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. Enforcement ----------- Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at dev@min.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. Attribution ----------- This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.4, available at [http://contributor-covenant.org/version/1/4](http://contributor-covenant.org/version/1/4/) This version includes a clarification to ensure that the code of conduct is in compliance with the free software licensing terms of the project. minio-go-7.0.97/constants.go000066400000000000000000000115631510244170000156770ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio // Multipart upload defaults. // absMinPartSize - absolute minimum part size (5 MiB) below which // a part in a multipart upload may not be uploaded. const absMinPartSize = 1024 * 1024 * 5 // minPartSize - minimum part size 16MiB per object after which // putObject behaves internally as multipart. const minPartSize = 1024 * 1024 * 16 // maxPartsCount - maximum number of parts for a single multipart session. const maxPartsCount = 10000 // maxPartSize - maximum part size 5GiB for a single multipart upload // operation. const maxPartSize = 1024 * 1024 * 1024 * 5 // maxSinglePutObjectSize - maximum size 5GiB of object per PUT // operation. const maxSinglePutObjectSize = 1024 * 1024 * 1024 * 5 // maxMultipartPutObjectSize - maximum size 5TiB of object for // Multipart operation. const maxMultipartPutObjectSize = 1024 * 1024 * 1024 * 1024 * 5 // unsignedPayload - value to be set to X-Amz-Content-Sha256 header when // we don't want to sign the request payload const unsignedPayload = "UNSIGNED-PAYLOAD" // unsignedPayloadTrailer value to be set to X-Amz-Content-Sha256 header when // we don't want to sign the request payload, but have a trailer. const unsignedPayloadTrailer = "STREAMING-UNSIGNED-PAYLOAD-TRAILER" // Total number of parallel workers used for multipart operation. const totalWorkers = 4 // Signature related constants. const ( signV4Algorithm = "AWS4-HMAC-SHA256" iso8601DateFormat = "20060102T150405Z" ) const ( // GetObjectAttributesTags are tags used to defined // return values for the GetObjectAttributes API GetObjectAttributesTags = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts" // GetObjectAttributesMaxParts defined the default maximum // number of parts returned by GetObjectAttributes GetObjectAttributesMaxParts = 1000 ) const ( // Response Headers // ETag is a common response header ETag = "ETag" // Storage class header. amzStorageClass = "X-Amz-Storage-Class" // Website redirect location header amzWebsiteRedirectLocation = "X-Amz-Website-Redirect-Location" // GetObjectAttributes headers amzPartNumberMarker = "X-Amz-Part-Number-Marker" amzExpectedBucketOnwer = "X-Amz-Expected-Bucket-Owner" amzMaxParts = "X-Amz-Max-Parts" amzObjectAttributes = "X-Amz-Object-Attributes" // Object Tagging headers amzTaggingHeader = "X-Amz-Tagging" amzTaggingHeaderDirective = "X-Amz-Tagging-Directive" amzVersionID = "X-Amz-Version-Id" amzTaggingCount = "X-Amz-Tagging-Count" amzExpiration = "X-Amz-Expiration" amzRestore = "X-Amz-Restore" amzReplicationStatus = "X-Amz-Replication-Status" amzDeleteMarker = "X-Amz-Delete-Marker" // Object legal hold header amzLegalHoldHeader = "X-Amz-Object-Lock-Legal-Hold" // Object retention header amzLockMode = "X-Amz-Object-Lock-Mode" amzLockRetainUntil = "X-Amz-Object-Lock-Retain-Until-Date" amzBypassGovernance = "X-Amz-Bypass-Governance-Retention" // Replication status amzBucketReplicationStatus = "X-Amz-Replication-Status" // Minio specific Replication/lifecycle transition extension minIOBucketSourceMTime = "X-Minio-Source-Mtime" minIOBucketSourceETag = "X-Minio-Source-Etag" minIOBucketReplicationDeleteMarker = "X-Minio-Source-DeleteMarker" minIOBucketReplicationProxyRequest = "X-Minio-Source-Proxy-Request" minIOBucketReplicationRequest = "X-Minio-Source-Replication-Request" minIOBucketReplicationCheck = "X-Minio-Source-Replication-Check" // Header indicates last tag update time on source minIOBucketReplicationTaggingTimestamp = "X-Minio-Source-Replication-Tagging-Timestamp" // Header indicates last retention update time on source minIOBucketReplicationObjectRetentionTimestamp = "X-Minio-Source-Replication-Retention-Timestamp" // Header indicates last legalhold update time on source minIOBucketReplicationObjectLegalHoldTimestamp = "X-Minio-Source-Replication-LegalHold-Timestamp" minIOForceDelete = "x-minio-force-delete" // Header indicates delete marker replication request can be sent by source now. minioTgtReplicationReady = "X-Minio-Replication-Ready" // Header asks if delete marker replication request can be sent by source now. isMinioTgtReplicationReady = "X-Minio-Check-Replication-Ready" ) minio-go-7.0.97/core.go000066400000000000000000000146161510244170000146150ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "io" "net/http" "github.com/minio/minio-go/v7/pkg/encrypt" ) // Core - Inherits Client and adds new methods to expose the low level S3 APIs. type Core struct { *Client } // NewCore - Returns new initialized a Core client, this CoreClient should be // only used under special conditions such as need to access lower primitives // and being able to use them to write your own wrappers. func NewCore(endpoint string, opts *Options) (*Core, error) { var s3Client Core client, err := New(endpoint, opts) if err != nil { return nil, err } s3Client.Client = client return &s3Client, nil } // ListObjects - List all the objects at a prefix, optionally with marker and delimiter // you can further filter the results. func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListBucketResult, err error) { return c.listObjectsQuery(context.Background(), bucket, prefix, marker, delimiter, maxKeys, nil) } // ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses // continuationToken instead of marker to support iteration over the results. func (c Core) ListObjectsV2(bucketName, objectPrefix, startAfter, continuationToken, delimiter string, maxkeys int) (ListBucketV2Result, error) { return c.listObjectsV2Query(context.Background(), bucketName, objectPrefix, continuationToken, true, false, delimiter, startAfter, maxkeys, nil) } // CopyObject - copies an object from source object to destination object on server side. func (c Core) CopyObject(ctx context.Context, sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string, srcOpts CopySrcOptions, dstOpts PutObjectOptions) (ObjectInfo, error) { return c.copyObjectDo(ctx, sourceBucket, sourceObject, destBucket, destObject, metadata, srcOpts, dstOpts) } // CopyObjectPart - creates a part in a multipart upload by copying (a // part of) an existing object. func (c Core) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, metadata map[string]string, ) (p CompletePart, err error) { return c.copyObjectPartDo(ctx, srcBucket, srcObject, destBucket, destObject, uploadID, partID, startOffset, length, metadata) } // PutObject - Upload object. Uploads using single PUT call. func (c Core) PutObject(ctx context.Context, bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, opts PutObjectOptions) (UploadInfo, error) { hookReader := newHook(data, opts.Progress) return c.putObjectDo(ctx, bucket, object, hookReader, md5Base64, sha256Hex, size, opts) } // NewMultipartUpload - Initiates new multipart upload and returns the new uploadID. func (c Core) NewMultipartUpload(ctx context.Context, bucket, object string, opts PutObjectOptions) (uploadID string, err error) { result, err := c.initiateMultipartUpload(ctx, bucket, object, opts) return result.UploadID, err } // ListMultipartUploads - List incomplete uploads. func (c Core) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartUploadsResult, err error) { return c.listMultipartUploadsQuery(ctx, bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads) } // PutObjectPartOptions contains options for PutObjectPart API type PutObjectPartOptions struct { Md5Base64, Sha256Hex string SSE encrypt.ServerSide CustomHeader, Trailer http.Header DisableContentSha256 bool } // PutObjectPart - Upload an object part. func (c Core) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data io.Reader, size int64, opts PutObjectPartOptions, ) (ObjectPart, error) { p := uploadPartParams{ bucketName: bucket, objectName: object, uploadID: uploadID, reader: data, partNumber: partID, md5Base64: opts.Md5Base64, sha256Hex: opts.Sha256Hex, size: size, sse: opts.SSE, streamSha256: !opts.DisableContentSha256, customHeader: opts.CustomHeader, trailer: opts.Trailer, } return c.uploadPart(ctx, p) } // ListObjectParts - List uploaded parts of an incomplete upload.x func (c Core) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int) (result ListObjectPartsResult, err error) { return c.listObjectPartsQuery(ctx, bucket, object, uploadID, partNumberMarker, maxParts) } // CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. func (c Core) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, parts []CompletePart, opts PutObjectOptions) (UploadInfo, error) { res, err := c.completeMultipartUpload(ctx, bucket, object, uploadID, completeMultipartUpload{ Parts: parts, }, opts) return res, err } // AbortMultipartUpload - Abort an incomplete upload. func (c Core) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error { return c.abortMultipartUpload(ctx, bucket, object, uploadID) } // GetBucketPolicy - fetches bucket access policy for a given bucket. func (c Core) GetBucketPolicy(ctx context.Context, bucket string) (string, error) { return c.getBucketPolicy(ctx, bucket) } // PutBucketPolicy - applies a new bucket access policy for a given bucket. func (c Core) PutBucketPolicy(ctx context.Context, bucket, bucketPolicy string) error { return c.putBucketPolicy(ctx, bucket, bucketPolicy) } // GetObject is a lower level API implemented to support reading // partial objects and also downloading objects with special conditions // matching etag, modtime etc. func (c Core) GetObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, http.Header, error) { return c.getObject(ctx, bucketName, objectName, opts) } minio-go-7.0.97/core_test.go000066400000000000000000000631001510244170000156440ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "context" "io" "math/rand" "net/http" "os" "strconv" "testing" "time" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" ) const ( serverEndpoint = "SERVER_ENDPOINT" accessKey = "ACCESS_KEY" secretKey = "SECRET_KEY" enableSecurity = "ENABLE_HTTPS" ) // Tests for Core GetObject() function. func TestGetObjectCore(t *testing.T) { if os.Getenv(serverEndpoint) == "" { t.Skip("SERVER_ENDPOINT not set") } if testing.Short() { t.Skip("skipping functional tests for the short runs") } // Instantiate new minio core client object. c, err := NewCore( os.Getenv(serverEndpoint), &Options{ Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), Secure: mustParseBool(os.Getenv(enableSecurity)), }) if err != nil { t.Fatal("Error:", err) } // Enable tracing, write to stderr. // c.TraceOn(os.Stderr) // Set user agent. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } // Generate data more than 32K buf := bytes.Repeat([]byte("3"), rand.Intn(1<<20)+32*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") _, err = c.Client.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), PutObjectOptions{ ContentType: "binary/octet-stream", }) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } st, err := c.StatObject(context.Background(), bucketName, objectName, StatObjectOptions{}) if err != nil { t.Fatal("Stat error:", err, bucketName, objectName) } if st.Size != int64(len(buf)) { t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size) } offset := int64(2048) // read directly buf1 := make([]byte, 512) buf2 := make([]byte, 512) buf3 := make([]byte, st.Size) buf4 := make([]byte, 1) opts := GetObjectOptions{} opts.SetRange(offset, offset+int64(len(buf1))-1) reader, objectInfo, _, err := c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { t.Fatal(err) } m, err := readFull(reader, buf1) reader.Close() if err != nil { t.Fatal(err) } if objectInfo.Size != int64(m) { t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) } if !bytes.Equal(buf1, buf[offset:offset+512]) { t.Fatal("Error: Incorrect read between two GetObject from same offset.") } offset += 512 opts.SetRange(offset, offset+int64(len(buf2))-1) reader, objectInfo, _, err = c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { t.Fatal(err) } m, err = readFull(reader, buf2) reader.Close() if err != nil { t.Fatal(err) } if objectInfo.Size != int64(m) { t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) } if !bytes.Equal(buf2, buf[offset:offset+512]) { t.Fatal("Error: Incorrect read between two GetObject from same offset.") } opts.SetRange(0, int64(len(buf3))) reader, objectInfo, _, err = c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { t.Fatal(err) } m, err = readFull(reader, buf3) if err != nil { reader.Close() t.Fatal(err) } reader.Close() if objectInfo.Size != int64(m) { t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) } if !bytes.Equal(buf3, buf) { t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") } opts = GetObjectOptions{} opts.SetMatchETag("etag") _, _, _, err = c.GetObject(context.Background(), bucketName, objectName, opts) if err == nil { t.Fatal("Unexpected GetObject should fail with mismatching etags") } if errResp := ToErrorResponse(err); errResp.Code != PreconditionFailed { t.Fatalf("Expected \"PreconditionFailed\" as code, got %s instead", errResp.Code) } opts = GetObjectOptions{} opts.SetMatchETagExcept("etag") reader, objectInfo, _, err = c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { t.Fatal(err) } m, err = readFull(reader, buf3) reader.Close() if err != nil { t.Fatal(err) } if objectInfo.Size != int64(m) { t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) } if !bytes.Equal(buf3, buf) { t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") } opts = GetObjectOptions{} opts.SetRange(0, 0) reader, objectInfo, _, err = c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { t.Fatal(err) } m, err = readFull(reader, buf4) reader.Close() if err != nil { t.Fatal(err) } if objectInfo.Size != int64(m) { t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) } opts = GetObjectOptions{} opts.SetRange(offset, offset+int64(len(buf2))-1) contentLength := len(buf2) var header http.Header _, _, header, err = c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { t.Fatal(err) } contentLengthValue, err := strconv.Atoi(header.Get("Content-Length")) if err != nil { t.Fatal("Error: ", err) } if contentLength != contentLengthValue { t.Fatalf("Error: Content Length in response header %v, not equal to set content length %v\n", contentLengthValue, contentLength) } err = c.RemoveObject(context.Background(), bucketName, objectName, RemoveObjectOptions{}) if err != nil { t.Fatal("Error: ", err) } err = c.RemoveBucket(context.Background(), bucketName) if err != nil { t.Fatal("Error:", err) } } // Tests GetObject to return Content-Encoding properly set // and overrides any auto decoding. func TestGetObjectContentEncoding(t *testing.T) { if os.Getenv(serverEndpoint) == "" { t.Skip("SERVER_ENDPOINT not set") } if testing.Short() { t.Skip("skipping functional tests for the short runs") } // Instantiate new minio core client object. c, err := NewCore( os.Getenv(serverEndpoint), &Options{ Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), Secure: mustParseBool(os.Getenv(enableSecurity)), }) if err != nil { t.Fatal("Error:", err) } // Enable tracing, write to stderr. // c.TraceOn(os.Stderr) // Set user agent. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } // Generate data more than 32K buf := bytes.Repeat([]byte("3"), rand.Intn(1<<20)+32*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") _, err = c.Client.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), PutObjectOptions{ ContentEncoding: "gzip", }) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } rwc, objInfo, _, err := c.GetObject(context.Background(), bucketName, objectName, GetObjectOptions{}) if err != nil { t.Fatalf("Error: %v", err) } rwc.Close() if objInfo.Size != int64(len(buf)) { t.Fatalf("Unexpected size of the object %v, expected %v", objInfo.Size, len(buf)) } value, ok := objInfo.Metadata["Content-Encoding"] if !ok { t.Fatalf("Expected Content-Encoding metadata to be set.") } if value[0] != "gzip" { t.Fatalf("Unexpected content-encoding found, want gzip, got %v", value) } err = c.RemoveObject(context.Background(), bucketName, objectName, RemoveObjectOptions{}) if err != nil { t.Fatal("Error: ", err) } err = c.RemoveBucket(context.Background(), bucketName) if err != nil { t.Fatal("Error:", err) } } // Tests get bucket policy core API. func TestGetBucketPolicy(t *testing.T) { if os.Getenv(serverEndpoint) == "" { t.Skip("SERVER_ENDPOINT not set") } if testing.Short() { t.Skip("skipping functional tests for short runs") } // Instantiate new minio client object. c, err := NewCore( os.Getenv(serverEndpoint), &Options{ Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), Secure: mustParseBool(os.Getenv(enableSecurity)), }) if err != nil { t.Fatal("Error:", err) } // Enable to debug // c.TraceOn(os.Stderr) // Set user agent. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } // Verify if bucket exits and you have access. var exists bool exists, err = c.BucketExists(context.Background(), bucketName) if err != nil { t.Fatal("Error:", err, bucketName) } if !exists { t.Fatal("Error: could not find ", bucketName) } // Asserting the default bucket policy. bucketPolicy, err := c.GetBucketPolicy(context.Background(), bucketName) if err != nil { errResp := ToErrorResponse(err) if errResp.Code != NoSuchBucketPolicy { t.Error("Error:", err, bucketName) } } if bucketPolicy != "" { t.Errorf("Bucket policy expected %#v, got %#v", "", bucketPolicy) } err = c.RemoveBucket(context.Background(), bucketName) if err != nil { t.Fatal("Error:", err) } } // Tests Core CopyObject API implementation. func TestCoreCopyObject(t *testing.T) { if os.Getenv(serverEndpoint) == "" { t.Skip("SERVER_ENDPOINT not set") } if testing.Short() { t.Skip("skipping functional tests for short runs") } // Instantiate new minio client object. c, err := NewCore( os.Getenv(serverEndpoint), &Options{ Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), Secure: mustParseBool(os.Getenv(enableSecurity)), }) if err != nil { t.Fatal("Error:", err) } // Enable tracing, write to stderr. // c.TraceOn(os.Stderr) // Set user agent. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } buf := bytes.Repeat([]byte("a"), 32*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") putopts := PutObjectOptions{ UserMetadata: map[string]string{ "Content-Type": "binary/octet-stream", }, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } st, err := c.StatObject(context.Background(), bucketName, objectName, StatObjectOptions{}) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } if st.Size != int64(len(buf)) { t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size) } destBucketName := bucketName destObjectName := objectName + "-dest" cuploadInfo, err := c.CopyObject(context.Background(), bucketName, objectName, destBucketName, destObjectName, map[string]string{ "X-Amz-Metadata-Directive": "REPLACE", "Content-Type": "application/javascript", }, CopySrcOptions{}, PutObjectOptions{}) if err != nil { t.Fatal("Error:", err, bucketName, objectName, destBucketName, destObjectName) } if cuploadInfo.ETag != uploadInfo.ETag { t.Fatalf("Error: expected etag to be same as source object %s, but found different etag %s", uploadInfo.ETag, cuploadInfo.ETag) } // Attempt to read from destBucketName and object name. r, err := c.Client.GetObject(context.Background(), destBucketName, destObjectName, GetObjectOptions{}) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } st, err = r.Stat() if err != nil { t.Fatal("Error:", err, bucketName, objectName) } if st.Size != int64(len(buf)) { t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", len(buf), st.Size) } if st.ContentType != "application/javascript" { t.Fatalf("Error: Content types don't match, expected: application/javascript, found: %+v\n", st.ContentType) } if st.ETag != uploadInfo.ETag { t.Fatalf("Error: expected etag to be same as source object %s, but found different etag :%s", uploadInfo.ETag, st.ETag) } if err := r.Close(); err != nil { t.Fatal("Error:", err) } if err := r.Close(); err == nil { t.Fatal("Error: object is already closed, should return error") } err = c.RemoveObject(context.Background(), bucketName, objectName, RemoveObjectOptions{}) if err != nil { t.Fatal("Error: ", err) } err = c.RemoveObject(context.Background(), destBucketName, destObjectName, RemoveObjectOptions{}) if err != nil { t.Fatal("Error: ", err) } err = c.RemoveBucket(context.Background(), bucketName) if err != nil { t.Fatal("Error:", err) } // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation func TestCoreCopyObjectPart(t *testing.T) { if os.Getenv(serverEndpoint) == "" { t.Skip("SERVER_ENDPOINT not set") } if testing.Short() { t.Skip("skipping functional tests for short runs") } // Instantiate new minio client object. c, err := NewCore( os.Getenv(serverEndpoint), &Options{ Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), Secure: mustParseBool(os.Getenv(enableSecurity)), }) if err != nil { t.Fatal("Error:", err) } // Enable tracing, write to stderr. // c.TraceOn(os.Stderr) // Set user agent. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) metadata := map[string]string{ "Content-Type": "binary/octet-stream", } putopts := PutObjectOptions{ UserMetadata: metadata, } // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } st, err := c.StatObject(context.Background(), bucketName, objectName, StatObjectOptions{}) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } if st.Size != int64(len(buf)) { t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size) } destBucketName := bucketName destObjectName := objectName + "-dest" uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, PutObjectOptions{}) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, nil) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, nil) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, nil) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []CompletePart{fstPart, sndPart, lstPart}, PutObjectOptions{}) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, StatObjectOptions{}) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } if objInfo.Size != (5*1024*1024)*2+1 { t.Fatal("Destination object has incorrect size!") } // Now we read the data back getOpts := GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } if !bytes.Equal(getBuf, buf) { t.Fatal("Got unexpected data in first 5MB") } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { t.Fatal("Error:", err, destBucketName, destObjectName) } if !bytes.Equal(getBuf[:5*1024*1024], buf) { t.Fatal("Got unexpected data in second 5MB") } if getBuf[5*1024*1024] != buf[0] { t.Fatal("Got unexpected data in last byte of copied object!") } if err := c.RemoveObject(context.Background(), destBucketName, destObjectName, RemoveObjectOptions{}); err != nil { t.Fatal("Error: ", err) } if err := c.RemoveObject(context.Background(), bucketName, objectName, RemoveObjectOptions{}); err != nil { t.Fatal("Error: ", err) } if err := c.RemoveBucket(context.Background(), bucketName); err != nil { t.Fatal("Error: ", err) } // Do not need to remove destBucketName its same as bucketName. } // Test Core PutObject. func TestCorePutObject(t *testing.T) { if os.Getenv(serverEndpoint) == "" { t.Skip("SERVER_ENDPOINT not set") } if testing.Short() { t.Skip("skipping functional tests for short runs") } // Instantiate new minio client object. c, err := NewCore( os.Getenv(serverEndpoint), &Options{ Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), Secure: mustParseBool(os.Getenv(enableSecurity)), }) if err != nil { t.Fatal("Error:", err) } // Enable tracing, write to stderr. // c.TraceOn(os.Stderr) // Set user agent. c.SetAppInfo("MinIO-go-FunctionalTest", "0.1.0") // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } buf := bytes.Repeat([]byte("a"), 32*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") // Object content type objectContentType := "binary/octet-stream" metadata := make(map[string]string) metadata["Content-Type"] = objectContentType putopts := PutObjectOptions{ UserMetadata: metadata, } _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "1B2M2Y8AsgTpgAmY7PhCfg==", "", putopts) if err == nil { t.Fatal("Error expected: error, got: nil(success)") } _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", putopts) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } // Read the data back r, err := c.Client.GetObject(context.Background(), bucketName, objectName, GetObjectOptions{}) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } st, err := r.Stat() if err != nil { t.Fatal("Error:", err, bucketName, objectName) } if st.Size != int64(len(buf)) { t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", len(buf), st.Size) } if st.ContentType != objectContentType { t.Fatalf("Error: Content types don't match, expected: %+v, found: %+v\n", objectContentType, st.ContentType) } if err := r.Close(); err != nil { t.Fatal("Error:", err) } if err := r.Close(); err == nil { t.Fatal("Error: object is already closed, should return error") } err = c.RemoveObject(context.Background(), bucketName, objectName, RemoveObjectOptions{}) if err != nil { t.Fatal("Error: ", err) } err = c.RemoveBucket(context.Background(), bucketName) if err != nil { t.Fatal("Error:", err) } } func TestCoreGetObjectMetadata(t *testing.T) { if os.Getenv(serverEndpoint) == "" { t.Skip("SERVER_ENDPOINT not set") } if testing.Short() { t.Skip("skipping functional tests for the short runs") } core, err := NewCore( os.Getenv(serverEndpoint), &Options{ Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), Secure: mustParseBool(os.Getenv(enableSecurity)), }) if err != nil { t.Fatal(err) } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = core.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } metadata := map[string]string{ "X-Amz-Meta-Key-1": "Val-1", } putopts := PutObjectOptions{ UserMetadata: metadata, } _, err = core.PutObject(context.Background(), bucketName, "my-objectname", bytes.NewReader([]byte("hello")), 5, "", "", putopts) if err != nil { t.Fatal(err) } reader, objInfo, _, err := core.GetObject(context.Background(), bucketName, "my-objectname", GetObjectOptions{}) if err != nil { t.Fatal(err) } reader.Close() if objInfo.Metadata.Get("X-Amz-Meta-Key-1") != "Val-1" { t.Fatal("Expected metadata to be available but wasn't") } err = core.RemoveObject(context.Background(), bucketName, "my-objectname", RemoveObjectOptions{}) if err != nil { t.Fatal("Error: ", err) } err = core.RemoveBucket(context.Background(), bucketName) if err != nil { t.Fatal("Error:", err) } } func TestCoreMultipartUpload(t *testing.T) { if os.Getenv(serverEndpoint) == "" { t.Skip("SERVER_ENDPOINT not set") } if testing.Short() { t.Skip("skipping functional tests for the short runs") } // Instantiate new minio client object. core, err := NewCore( os.Getenv(serverEndpoint), &Options{ Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), Secure: mustParseBool(os.Getenv(enableSecurity)), }) if err != nil { t.Fatal("Error:", err) } bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = core.MakeBucket(context.Background(), bucketName, MakeBucketOptions{Region: "us-east-1"}) if err != nil { t.Fatal("Error:", err, bucketName) } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") objectContentType := "binary/octet-stream" metadata := make(map[string]string) metadata["Content-Type"] = objectContentType putopts := PutObjectOptions{ UserMetadata: metadata, } uploadID, err := core.NewMultipartUpload(context.Background(), bucketName, objectName, putopts) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } buf := bytes.Repeat([]byte("a"), 32*1024*1024) r := bytes.NewReader(buf) partBuf := make([]byte, 100*1024*1024) parts := make([]CompletePart, 0, 5) partID := 0 for { n, err := r.Read(partBuf) if err != nil && err != io.EOF { t.Fatal("Error:", err) } if err == io.EOF { break } if n > 0 { partID++ data := bytes.NewReader(partBuf[:n]) dataLen := int64(len(partBuf[:n])) objectPart, err := core.PutObjectPart(context.Background(), bucketName, objectName, uploadID, partID, data, dataLen, PutObjectPartOptions{ Md5Base64: "", Sha256Hex: "", SSE: encrypt.NewSSE(), CustomHeader: nil, Trailer: nil, }, ) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } parts = append(parts, CompletePart{ PartNumber: partID, ETag: objectPart.ETag, }) } } objectParts, err := core.listObjectParts(context.Background(), bucketName, objectName, uploadID) if err != nil { t.Fatal("Error:", err) } if len(objectParts) != len(parts) { t.Fatal("Error", len(objectParts), len(parts)) } _, err = core.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, putopts) if err != nil { t.Fatal("Error:", err) } if err := core.RemoveObject(context.Background(), bucketName, objectName, RemoveObjectOptions{}); err != nil { t.Fatal("Error: ", err) } if err := core.RemoveBucket(context.Background(), bucketName); err != nil { t.Fatal("Error: ", err) } } minio-go-7.0.97/create-session.go000066400000000000000000000132131510244170000166010ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2025 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "encoding/xml" "errors" "net" "net/http" "net/url" "path" "time" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/s3utils" "github.com/minio/minio-go/v7/pkg/signer" ) // SessionMode - session mode type there are only two types type SessionMode string // Session constants const ( SessionReadWrite SessionMode = "ReadWrite" SessionReadOnly SessionMode = "ReadOnly" ) type createSessionResult struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreateSessionResult"` Credentials struct { AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"` SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"` SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"` Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"` } `xml:",omitempty"` } // CreateSession - https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateSession.html // the returning credentials may be cached depending on the expiration of the original // credential, credentials will get renewed 10 secs earlier than when its gonna expire // allowing for some leeway in the renewal process. func (c *Client) CreateSession(ctx context.Context, bucketName string, sessionMode SessionMode) (cred credentials.Value, err error) { if err := s3utils.CheckValidBucketNameS3Express(bucketName); err != nil { return credentials.Value{}, err } v, ok := c.bucketSessionCache.Get(bucketName) if ok && v.Expiration.After(time.Now().Add(10*time.Second)) { // Verify if the credentials will not expire // in another 10 seconds, if not we renew it again. return v, nil } req, err := c.createSessionRequest(ctx, bucketName, sessionMode) if err != nil { return credentials.Value{}, err } resp, err := c.do(req) defer closeResponse(resp) if err != nil { return credentials.Value{}, err } if resp.StatusCode != http.StatusOK { return credentials.Value{}, httpRespToErrorResponse(resp, bucketName, "") } credSession := &createSessionResult{} dec := xml.NewDecoder(resp.Body) if err = dec.Decode(credSession); err != nil { return credentials.Value{}, err } defer c.bucketSessionCache.Set(bucketName, cred) return credentials.Value{ AccessKeyID: credSession.Credentials.AccessKey, SecretAccessKey: credSession.Credentials.SecretKey, SessionToken: credSession.Credentials.SessionToken, Expiration: credSession.Credentials.Expiration, }, nil } // createSessionRequest - Wrapper creates a new CreateSession request. func (c *Client) createSessionRequest(ctx context.Context, bucketName string, sessionMode SessionMode) (*http.Request, error) { // Set location query. urlValues := make(url.Values) urlValues.Set("session", "") // Set get bucket location always as path style. targetURL := *c.endpointURL // Fetch new host based on the bucket location. host := getS3ExpressEndpoint(c.region, s3utils.IsS3ExpressBucket(bucketName)) // as it works in makeTargetURL method from api.go file if h, p, err := net.SplitHostPort(host); err == nil { if targetURL.Scheme == "http" && p == "80" || targetURL.Scheme == "https" && p == "443" { host = h if ip := net.ParseIP(h); ip != nil && ip.To4() == nil { host = "[" + h + "]" } } } isVirtualStyle := c.isVirtualHostStyleRequest(targetURL, bucketName) var urlStr string if isVirtualStyle { urlStr = c.endpointURL.Scheme + "://" + bucketName + "." + host + "/?session" } else { targetURL.Path = path.Join(bucketName, "") + "/" targetURL.RawQuery = urlValues.Encode() urlStr = targetURL.String() } // Get a new HTTP request for the method. req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil) if err != nil { return nil, err } // Set UserAgent for the request. c.setUserAgent(req) // Get credentials from the configured credentials provider. value, err := c.credsProvider.GetWithContext(c.CredContext()) if err != nil { return nil, err } var ( signerType = value.SignerType accessKeyID = value.AccessKeyID secretAccessKey = value.SecretAccessKey sessionToken = value.SessionToken ) // Custom signer set then override the behavior. if c.overrideSignerType != credentials.SignatureDefault { signerType = c.overrideSignerType } // If signerType returned by credentials helper is anonymous, // then do not sign regardless of signerType override. if value.SignerType == credentials.SignatureAnonymous { signerType = credentials.SignatureAnonymous } if signerType.IsAnonymous() || signerType.IsV2() { return req, errors.New("Only signature v4 is supported for CreateSession() API") } // Set sha256 sum for signature calculation only with signature version '4'. contentSha256 := emptySHA256Hex if c.secure { contentSha256 = unsignedPayload } req.Header.Set("X-Amz-Content-Sha256", contentSha256) req.Header.Set("x-amz-create-session-mode", string(sessionMode)) req = signer.SignV4Express(*req, accessKeyID, secretAccessKey, sessionToken, c.region) return req, nil } minio-go-7.0.97/docs/000077500000000000000000000000001510244170000142565ustar00rootroot00000000000000minio-go-7.0.97/docs/API.md000066400000000000000000004431401510244170000152170ustar00rootroot00000000000000MinIO Go Client API Reference [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) =================================================================================================== Initialize MinIO Client object. ------------------------------- MinIO ----- ```go package main import ( "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { endpoint := "play.min.io" accessKeyID := "Q3AM3UQ867SPQQA43P2F" secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" useSSL := true // Initialize minio client object. minioClient, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), Secure: useSSL, }) if err != nil { log.Fatalln(err) } log.Printf("%#v\n", minioClient) // minioClient is now setup } ``` AWS S3 ------ ```go package main import ( "fmt" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Initialize minio client object. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { fmt.Println(err) return } } ``` | Bucket operations | Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | |:--------------------------------------------------------------|:----------------------------------------------------|:----------------------------------------------|:--------------------------------------------------------------|:------------------------------------------------------| | [`MakeBucket`](#MakeBucket) | [`AppendObject`](#AppendObject) | [`PresignedGetObject`](#PresignedGetObject) | [`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | | [`ListBuckets`](#ListBuckets) | [`GetObject`](#GetObject) | [`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`TraceOn`](#TraceOn) | | [`BucketExists`](#BucketExists) | [`PutObject`](#PutObject) | [`PresignedHeadObject`](#PresignedHeadObject) | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOff`](#TraceOff) | | [`RemoveBucket`](#RemoveBucket) | [`PutObjectFanOut`](#PutObjectFanOut) | [`PresignedPostPolicy`](#PresignedPostPolicy) | [`GetBucketNotification`](#GetBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | | [`ListObjects`](#ListObjects) | [`CopyObject`](#CopyObject) | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | | | [`ListIncompleteUploads`](#ListIncompleteUploads) | [`ComposeObject`](#ComposeObject) | | [`ListenBucketNotification`](#ListenBucketNotification) | | | [`SetBucketTagging`](#SetBucketTagging) | [`StatObject`](#StatObject) | | [`ListenNotification`](#ListenNotification) | | | [`GetBucketTagging`](#GetBucketTagging) | [`RemoveObject`](#RemoveObject) | | [`SetBucketLifecycle`](#SetBucketLifecycle) | | | [`RemoveBucketTagging`](#RemoveBucketTagging) | [`RemoveObjects`](#RemoveObjects) | | [`GetBucketLifecycle`](#GetBucketLifecycle) | | | [`SetBucketCors`](#SetBucketCors) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | [`SetBucketEncryption`](#SetBucketEncryption) | | | [`GetBucketCors`](#GetBucketCors) | [`FPutObject`](#FPutObject) | | [`GetBucketEncryption`](#GetBucketEncryption) | | | [`SetBucketReplication`](#SetBucketReplication) | [`FGetObject`](#FGetObject) | | [`RemoveBucketEncryption`](#RemoveBucketEncryption) | | | [`GetBucketReplication`](#GetBucketReplication) | [`PutObjectRetention`](#PutObjectRetention) | | [`SetObjectLockConfig`](#SetObjectLockConfig) | | | [`RemoveBucketReplication`](#RemoveBucketReplication) | [`GetObjectRetention`](#GetObjectRetention) | | [`GetObjectLockConfig`](#GetObjectLockConfig) | | | [`GetBucketReplicationMetrics`](#GetBucketReplicationMetrics) | [`PutObjectLegalHold`](#PutObjectLegalHold) | | [`EnableVersioning`](#EnableVersioning) | | | [`GetBucketLocation`](#GetBucketLocation) | [`GetObjectLegalHold`](#GetObjectLegalHold) | | [`SuspendVersioning`](#SuspendVersioning) | | | | [`SelectObjectContent`](#SelectObjectContent) | | [`GetBucketVersioning`](#GetBucketVersioning) | | | | [`PutObjectTagging`](#PutObjectTagging) | | | | | | [`GetObjectTagging`](#GetObjectTagging) | | | | | | [`RemoveObjectTagging`](#RemoveObjectTagging) | | | | | | [`RestoreObject`](#RestoreObject) | | | | | | [`GetObjectAttributes`](#GetObjectAttributes) | | | | | | [`PromptObject`](#PromptObject) | | | | 1. Constructor -------------- ### New(endpoint string, opts \*Options) (\*Client, error) Initializes a new client object. **Parameters** | Param | Type | Description | |:-----------|:----------------|:--------------------------------------| | `endpoint` | *string* | S3 compatible object storage endpoint | | `opts` | *minio.Options* | Options for constructing a new client | **minio.Options** | Field | Type | Description | |:--------------------|:----------------------------|:-----------------------------------------------------------------------------| | `opts.Creds` | \**credentials.Credentials* | S3 compatible object storage access credentials | | `opts.Secure` | *bool* | If 'true' API requests will be secure (HTTPS), and insecure (HTTP) otherwise | | `opts.Transport` | *http.RoundTripper* | Custom transport for executing HTTP transactions | | `opts.Region` | *string* | S3 compatible object storage region | | `opts.BucketLookup` | *BucketLookupType* | Bucket lookup type can be one of the following values | | | | *minio.BucketLookupDNS* | | | | *minio.BucketLookupPath* | | | | *minio.BucketLookupAuto* | 1. Bucket operations -------------------- ### MakeBucket(ctx context.Context, bucketName string, opts MakeBucketOptions) error Creates a new bucket. **Parameters** | Param | Type | Description | |--------------|---------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `opts` | *minio.MakeBucketOptions* | Bucket options such as `Region` where the bucket is to be created. Default value is us-east-1. Other valid values are listed below. Note: When used with minio server, use the region specified in its config file (defaults to us-east-1). | | | | us-east-1 | | | | us-east-2 | | | | us-west-1 | | | | us-west-2 | | | | ca-central-1 | | | | eu-west-1 | | | | eu-west-2 | | | | eu-west-3 | | | | eu-central-1 | | | | eu-north-1 | | | | ap-east-1 | | | | ap-south-1 | | | | ap-southeast-1 | | | | ap-southeast-2 | | | | ap-northeast-1 | | | | ap-northeast-2 | | | | ap-northeast-3 | | | | me-south-1 | | | | sa-east-1 | | | | us-gov-west-1 | | | | us-gov-east-1 | | | | cn-north-1 | | | | cn-northwest-1 | **Example** ```go // Create a bucket at region 'us-east-1' with object locking enabled. err = minioClient.MakeBucket(context.Background(), "mybucket", minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { fmt.Println(err) return } fmt.Println("Successfully created mybucket.") ``` ### ListBuckets(ctx context.Context) ([]BucketInfo, error) Lists all buckets. | Param | Type | Description | |--------------|----------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketList` | *[]minio.BucketInfo* | Lists of all buckets | **minio.BucketInfo** | Field | Type | Description | |-----------------------|-------------|-------------------------| | `bucket.Name` | *string* | Name of the bucket | | `bucket.CreationDate` | *time.Time* | Date of bucket creation | **Example** ```go buckets, err := minioClient.ListBuckets(context.Background()) if err != nil { fmt.Println(err) return } for _, bucket := range buckets { fmt.Println(bucket) } ``` ### BucketExists(ctx context.Context, bucketName string) (found bool, err error) Checks if a bucket exists. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:--------|:--------|:---------------------------------------| | `found` | *bool* | Indicates whether bucket exists or not | | `err` | *error* | Standard Error | **Example** ```go found, err := minioClient.BucketExists(context.Background(), "mybucket") if err != nil { fmt.Println(err) return } if found { fmt.Println("Bucket found") } ``` ### RemoveBucket(ctx context.Context, bucketName string) error Removes a bucket, bucket should be empty to be successfully removed. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Example** ```go err = minioClient.RemoveBucket(context.Background(), "mybucket") if err != nil { fmt.Println(err) return } ``` ### ListObjects(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo Lists objects in a bucket. **Parameters** | Param | Type | Description | |:-------------|:---------------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `opts` | *minio.ListObjectsOptions* | Options per to list objects | **Return Value** | Param | Type | Description | |:-------------|:------------------------|:--------------------------------------------------------------------------------------| | `objectInfo` | *chan minio.ObjectInfo* | Read channel for all objects in the bucket, the object is of the format listed below: | **minio.ObjectInfo** | Field | Type | Description | |:--------------------------|:------------|:-----------------------------------| | `objectInfo.Key` | *string* | Name of the object | | `objectInfo.Size` | *int64* | Size of the object | | `objectInfo.ETag` | *string* | MD5 checksum of the object | | `objectInfo.LastModified` | *time.Time* | Time when object was last modified | ```go ctx, cancel := context.WithCancel(context.Background()) defer cancel() objectCh := minioClient.ListObjects(ctx, "mybucket", minio.ListObjectsOptions{ Prefix: "myprefix", Recursive: true, }) for object := range objectCh { if object.Err != nil { fmt.Println(object.Err) return } fmt.Println(object) } ``` ### ListObjectsIter(ctx context.Context, bucketName string, opts ListObjectsOptions) iter.Seq[ObjectInfo] Lists objects in a bucket using an iterator. This is a modern Go 1.23+ alternative to the channel-based ListObjects API. **Parameters** | Param | Type | Description | |:-------------|:---------------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `opts` | *minio.ListObjectsOptions* | Options to list objects | **Return Value** | Param | Type | Description | |:-----------|:-----------------------------|:------------------------------------| | `iterator` | *iter.Seq[minio.ObjectInfo]* | Iterator yielding ObjectInfo values | **Example** ```go opts := minio.ListObjectsOptions{ Prefix: "myprefix", Recursive: true, } for object := range minioClient.ListObjectsIter(context.Background(), "mybucket", opts) { if object.Err != nil { fmt.Println(object.Err) return } fmt.Println(object.Key, object.Size) } ``` ### ListIncompleteUploads(ctx context.Context, bucketName, prefix string, recursive bool) <- chan ObjectMultipartInfo Lists partially uploaded objects in a bucket. **Parameters** | Param | Type | Description | |:-------------|:------------------|:---------------------------------------------------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `prefix` | *string* | Prefix of objects that are partially uploaded | | `recursive` | *bool* | `true` indicates recursive style listing and `false` indicates directory style listing delimited by '/'. | **Return Value** | Param | Type | Description | |:----------------|:---------------------------------|:----------------------------------------------------| | `multiPartInfo` | *chan minio.ObjectMultipartInfo* | Emits multipart objects of the format listed below: | **minio.ObjectMultipartInfo** | Field | Type | Description | |:----------------------------|:---------|:------------------------------------------| | `multiPartObjInfo.Key` | *string* | Name of incompletely uploaded object | | `multiPartObjInfo.UploadID` | *string* | Upload ID of incompletely uploaded object | | `multiPartObjInfo.Size` | *int64* | Size of incompletely uploaded object | **Example** ```go isRecursive := true // Recursively list everything at 'myprefix' multiPartObjectCh := minioClient.ListIncompleteUploads(context.Background(), "mybucket", "myprefix", isRecursive) for multiPartObject := range multiPartObjectCh { if multiPartObject.Err != nil { fmt.Println(multiPartObject.Err) return } fmt.Println(multiPartObject) } ``` ### SetBucketTagging(ctx context.Context, bucketName string, tags *tags.Tags) error Sets tags to a bucket. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `tags` | \**tags.Tags* | Bucket tags | **Example** ```go // Create tags from a map. tags, err := tags.NewTags(map[string]string{ "Tag1": "Value1", "Tag2": "Value2", }, false) if err != nil { log.Fatalln(err) } err = minioClient.SetBucketTagging(context.Background(), "my-bucketname", tags) if err != nil { log.Fatalln(err) } ``` ### GetBucketTagging(ctx context.Context, bucketName string) (*tags.Tags, error) Gets tags of a bucket. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Value** | Param | Type | Description | |:-------|:--------------|:------------| | `tags` | \**tags.Tags* | Bucket tags | **Example** ```go tags, err := minioClient.GetBucketTagging(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } fmt.Printf("Fetched Object Tags: %v\n", tags) ``` ### RemoveBucketTagging(ctx context.Context, bucketName string) error Removes all tags on a bucket. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Example** ```go err := minioClient.RemoveBucketTagging(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } ``` 1. Object operations -------------------- ### AppendObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, opts AppendObjectOptions) (UploadInfo, error) **Parameters** |Param | Type | Description | |:--- | :--- | :--- | |`ctx` | *context.Context* | Custom Context for timeout/cancellation of the call| |`bucketName`| *string* | Name of bucket | |`objectName`| *string* | Name of Object | |`reader` | *io.Reader* | standard Reader Interface | |`objectSize` | *int64* | Size of the object | |`opts` | *minio.AppendObjectOptions* | Additional Options for Append Operation| **Return Value** |Param | Type | Description | |:--- | :--- | :--- | |`info`| *minio.UploadInfo* | Information about the newly uploaded or copied object | |`err`| *error* | Standard error | **minio.AppendObjectOptions** | Field | Type | Description | |:--- | :--- | :--- | |`opts.Progress`| *io.Reader* | A progress reader to indicate progress| |`opts.ChunkSize`| *uint64* | Maximum Append Size | |`opts.DisableContentSha256`| *bool* | Aggressively disable sha256 payload. | **minio.UploadInfo** | Field | Type | Description | | :--- | :--- | :--- | | `info.Bucket` | *string* | Name of bucket | | `info.Key` | *string* | Name of object | | `info.ETag` | *string* | MD5 checksum of the object | | `info.Size` | *string* | Size of object | **Example** ```go opt := minio.AppendObjectOptions{} info, err := minio.AppendObject(context.Background(), "my-bucket-name", "my-object-name", my_progress_reader, size, opt) if err != nil { log.Fatalln(err) } ``` ### GetObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) Returns a stream of the object data. Most of the common errors occur when reading the stream. **Parameters** | Param | Type | Description | |:-------------|:-------------------------|:---------------------------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `opts` | *minio.GetObjectOptions* | Options for GET requests specifying additional options like encryption, If-Match | **minio.GetObjectOptions** | Field | Type | Description | |:----------------------------|:---------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------| | `opts.ServerSideEncryption` | *encrypt.ServerSide* | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v7\) | | `opts.Internal` | *minio.AdvancedGetOptions* | This option is intended for internal use by MinIO server. This option should not be set unless the application is aware of intended use. | **Return Value** | Param | Type | Description | |:---------|:-----------------|:-------------------------------------------------------------------------------------------------------------------| | `object` | \**minio.Object* | *minio.Object* represents object reader. It implements io.Reader, io.Seeker, io.ReaderAt and io.Closer interfaces. | **Example** ```go object, err := minioClient.GetObject(context.Background(), "mybucket", "myobject", minio.GetObjectOptions{}) if err != nil { fmt.Println(err) return } defer object.Close() localFile, err := os.Create("/tmp/local-file.jpg") if err != nil { fmt.Println(err) return } defer localFile.Close() if _, err = io.Copy(localFile, object); err != nil { fmt.Println(err) return } ``` ### FGetObject(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error Downloads and saves the object as a file in the local filesystem. **Parameters** | Param | Type | Description | |:-------------|:-------------------------|:---------------------------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `filePath` | *string* | Path to download object to | | `opts` | *minio.GetObjectOptions* | Options for GET requests specifying additional options like encryption, If-Match | **Example** ```go err = minioClient.FGetObject(context.Background(), "mybucket", "myobject", "/tmp/myobject", minio.GetObjectOptions{}) if err != nil { fmt.Println(err) return } ``` ### PutObjectFanOut(ctx context.Context, bucket string, body io.Reader, fanOutReq ...PutObjectFanOutRequest) ([]PutObjectFanOutResponse, error) A variant of PutObject instead of writing a single object from a single stream multiple objects are written, defined via a list of *PutObjectFanOutRequest*. Each entry in *PutObjectFanOutRequest* carries an object keyname and its relevant metadata if any. `Key` is mandatory, rest of the other options in *PutObjectFanOutRequest( are optional. **Parameters** | Param | Type | Description | |:-------------|:-------------------------------|:----------------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `fanOutData` | *io.Reader* | Any Go type that implements io.Reader | | `fanOutReq` | *minio.PutObjectFanOutRequest* | User input list of all the objects that will be created on the server | | | | | **minio.PutObjectFanOutRequest** | Field | Type | Description | |:------------|:--------------------------------|:-------------------------------------------| | `Entries` | *[]minio.PutObjectFanOutEntyry* | List of object fan out entries | | `Checksums` | *map[string]string* | Checksums for the input data | | `SSE` | _encrypt.ServerSide | Encryption settings for the entire fan-out | **minio.PutObjectFanOutEntry** | Field | Type | Description | |:---------------------|:----------------------|:---------------------------------------------------------------------------------------------------| | `Key` | *string* | Name of the object | | `UserMetadata` | *map[string]string* | Map of user metadata | | `UserTags` | *map[string]string* | Map of user object tags | | `ContentType` | *string* | Content type of object, e.g "application/text" | | `ContentEncoding` | *string* | Content encoding of object, e.g "gzip" | | `ContentDisposition` | *string* | Content disposition of object, "inline" | | `ContentLanguage` | *string* | Content language of object, e.g "French" | | `CacheControl` | *string* | Used to specify directives for caching mechanisms in both requests and responses e.g "max-age=600" | | `Retention` | *minio.RetentionMode* | Retention mode to be set, e.g "COMPLIANCE" | | `RetainUntilDate` | *time.Time* | Time until which the retention applied is valid | **minio.PutObjectFanOutResponse** | Field | Type | Description | |:---------------|:-----------|:----------------------------------------------------------------| | `Key` | *string* | Name of the object | | `ETag` | *string* | ETag opaque unique value of the object | | `VersionID` | *string* | VersionID of the uploaded object | | `LastModified` | _time.Time | Last modified time of the latest object | | `Error` | *error* | Is non `nil` only when the fan-out for a specific object failed | ### PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64,opts PutObjectOptions) (info UploadInfo, err error) Uploads objects that are less than 128MiB in a single PUT operation. For objects that are greater than 128MiB in size, PutObject seamlessly uploads the object as parts of 128MiB or more depending on the actual file size. The max upload size for an object is 5TB. **Parameters** | Param | Type | Description | |:-------------|:-------------------------|:------------------------------------------------------------------------------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `reader` | *io.Reader* | Any Go type that implements io.Reader | | `objectSize` | *int64* | Size of the object being uploaded. Pass -1 if stream size is unknown (Warning: passing -1 will allocate a large amount of memory) | | `opts` | *minio.PutObjectOptions* | Allows user to set optional custom metadata, content headers, encryption keys and number of threads for multipart upload operation. | **minio.PutObjectOptions** | Field | Type | Description | |:-------------------------------|:---------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `opts.UserMetadata` | *map[string]string* | Map of user metadata | | `opts.UserTags` | *map[string]string* | Map of user object tags | | `opts.Progress` | *io.Reader* | Reader to fetch progress of an upload | | `opts.ContentType` | *string* | Content type of object, e.g "application/text" | | `opts.ContentEncoding` | *string* | Content encoding of object, e.g "gzip" | | `opts.ContentDisposition` | *string* | Content disposition of object, "inline" | | `opts.ContentLanguage` | *string* | Content language of object, e.g "French" | | `opts.CacheControl` | *string* | Used to specify directives for caching mechanisms in both requests and responses e.g "max-age=600" | | `opts.Mode` | \**minio.RetentionMode* | Retention mode to be set, e.g "COMPLIANCE" | | `opts.RetainUntilDate` | \**time.Time* | Time until which the retention applied is valid | | `opts.ServerSideEncryption` | *encrypt.ServerSide* | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v7\) | | `opts.StorageClass` | *string* | Specify storage class for the object. Supported values for MinIO server are `REDUCED_REDUNDANCY` and `STANDARD` | | `opts.WebsiteRedirectLocation` | *string* | Specify a redirect for the object, to another object in the same bucket or to a external URL. | | `opts.SendContentMd5` | *bool* | Specify if you'd like to send `content-md5` header with PutObject operation. Note that setting this flag will cause higher memory usage because of in-memory `md5sum` calculation. | | `opts.PartSize` | *uint64* | Specify a custom part size used for uploading the object | | `opts.Internal` | *minio.AdvancedPutOptions* | This option is intended for internal use by MinIO server and should not be set unless the application is aware of intended use. | | | | | **minio.UploadInfo** | Field | Type | Description | |:-----------------|:---------|:-----------------------------------------| | `info.ETag` | *string* | The ETag of the new object | | `info.VersionID` | *string* | The version identifier of the new object | **Example** ```go file, err := os.Open("my-testfile") if err != nil { fmt.Println(err) return } defer file.Close() fileStat, err := file.Stat() if err != nil { fmt.Println(err) return } uploadInfo, err := minioClient.PutObject(context.Background(), "mybucket", "myobject", file, fileStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { fmt.Println(err) return } fmt.Println("Successfully uploaded bytes: ", uploadInfo) ``` API methods PutObjectWithSize, PutObjectWithMetadata, PutObjectStreaming, and PutObjectWithProgress available in minio-go SDK release v3.0.3 are replaced by the new PutObject call variant that accepts a pointer to PutObjectOptions struct. ### CopyObject(ctx context.Context, dst CopyDestOptions, src CopySrcOptions) (UploadInfo, error) Create or replace an object through server-side copying of an existing object. It supports conditional copying, copying a part of an object and server-side encryption of destination and decryption of source. See the `CopySrcOptions` and `DestinationInfo` types for further details. To copy multiple source objects into a single destination object see the `ComposeObject` API. **Parameters** | Param | Type | Description | |:------|:------------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `dst` | *minio.CopyDestOptions* | Argument describing the destination object | | `src` | *minio.CopySrcOptions* | Argument describing the source object | **minio.UploadInfo** | Field | Type | Description | |:-----------------|:---------|:-----------------------------------------| | `info.ETag` | *string* | The ETag of the new object | | `info.VersionID` | *string* | The version identifier of the new object | **Example** ```go // Use-case 1: Simple copy object with no conditions. // Source object srcOpts := minio.CopySrcOptions{ Bucket: "my-sourcebucketname", Object: "my-sourceobjectname", } // Destination object dstOpts := minio.CopyDestOptions{ Bucket: "my-bucketname", Object: "my-objectname", } // Copy object call uploadInfo, err := minioClient.CopyObject(context.Background(), dstOpts, srcOpts) if err != nil { fmt.Println(err) return } fmt.Println("Successfully copied object:", uploadInfo) ``` ```go // Use-case 2: // Copy object with copy-conditions, and copying only part of the source object. // 1. that matches a given ETag // 2. and modified after 1st April 2014 // 3. but unmodified since 23rd April 2014 // 4. copy only first 1MiB of object. // Source object srcOpts := minio.CopySrcOptions{ Bucket: "my-sourcebucketname", Object: "my-sourceobjectname", MatchETag: "31624deb84149d2f8ef9c385918b653a", MatchModifiedSince: time.Date(2014, time.April, 1, 0, 0, 0, 0, time.UTC), MatchUnmodifiedSince: time.Date(2014, time.April, 23, 0, 0, 0, 0, time.UTC), Start: 0, End: 1024*1024 - 1, } // Destination object dstOpts := minio.CopyDestOptions{ Bucket: "my-bucketname", Object: "my-objectname", } // Copy object call _, err = minioClient.CopyObject(context.Background(), dstOpts, srcOpts) if err != nil { fmt.Println(err) return } fmt.Println("Successfully copied object:", uploadInfo) ``` ### ComposeObject(ctx context.Context, dst minio.CopyDestOptions, srcs ...minio.CopySrcOptions) (UploadInfo, error) Create an object by concatenating a list of source objects using server-side copying. **Parameters** | Param | Type | Description | |:-------|:--------------------------|:----------------------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `dst` | *minio.CopyDestOptions* | Struct with info about the object to be created. | | `srcs` | *...minio.CopySrcOptions* | Slice of struct with info about source objects to be concatenated in order. | **minio.UploadInfo** | Field | Type | Description | |:-----------------|:---------|:-----------------------------------------| | `info.ETag` | *string* | The ETag of the new object | | `info.VersionID` | *string* | The version identifier of the new object | **Example** ```go // Prepare source decryption key (here we assume same key to // decrypt all source objects.) sseSrc := encrypt.DefaultPBKDF([]byte("password"), []byte("salt")) // Source objects to concatenate. We also specify decryption // key for each src1Opts := minio.CopySrcOptions{ Bucket: "bucket1", Object: "object1", Encryption: sseSrc, MatchETag: "31624deb84149d2f8ef9c385918b653a", } src2Opts := minio.CopySrcOptions{ Bucket: "bucket2", Object: "object2", Encryption: sseSrc, MatchETag: "f8ef9c385918b653a31624deb84149d2", } src3Opts := minio.CopySrcOptions{ Bucket: "bucket3", Object: "object3", Encryption: sseSrc, MatchETag: "5918b653a31624deb84149d2f8ef9c38", } // Prepare destination encryption key sseDst := encrypt.DefaultPBKDF([]byte("new-password"), []byte("new-salt")) // Create destination info dstOpts := CopyDestOptions{ Bucket: "bucket", Object: "object", Encryption: sseDst, } // Compose object call by concatenating multiple source files. uploadInfo, err := minioClient.ComposeObject(context.Background(), dst, srcs...) if err != nil { fmt.Println(err) return } fmt.Println("Composed object successfully:", uploadInfo) ``` ### FPutObject(ctx context.Context, bucketName, objectName, filePath string, opts PutObjectOptions) (info UploadInfo, err error) Uploads contents from a file to objectName. FPutObject uploads objects that are less than 128MiB in a single PUT operation. For objects that are greater than the 128MiB in size, FPutObject seamlessly uploads the object in chunks of 128MiB or more depending on the actual file size. The max upload size for an object is 5TB. **Parameters** | Param | Type | Description | |:-------------|:-------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `filePath` | *string* | Path to file to be uploaded | | `opts` | *minio.PutObjectOptions* | Pointer to struct that allows user to set optional custom metadata, content-type, content-encoding, content-disposition, content-language and cache-control headers, pass encryption module for encrypting objects, and optionally configure number of threads for multipart put operation. | **minio.UploadInfo** | Field | Type | Description | |:-----------------|:---------|:-----------------------------------------| | `info.ETag` | *string* | The ETag of the new object | | `info.VersionID` | *string* | The version identifier of the new object | **Example** ```go uploadInfo, err := minioClient.FPutObject(context.Background(), "my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ ContentType: "application/csv", }) if err != nil { fmt.Println(err) return } fmt.Println("Successfully uploaded object: ", uploadInfo) ``` ### StatObject(ctx context.Context, bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) Fetch metadata of an object. **Parameters** | Param | Type | Description | |:-------------|:--------------------------|:-------------------------------------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `opts` | *minio.StatObjectOptions* | Options for GET info/stat requests specifying additional options like encryption, If-Match | **Return Value** | Param | Type | Description | |:----------|:-------------------|:------------------------| | `objInfo` | *minio.ObjectInfo* | Object stat information | **minio.ObjectInfo** | Field | Type | Description | |:-----------------------|:------------|:-----------------------------------| | `objInfo.LastModified` | *time.Time* | Time when object was last modified | | `objInfo.ETag` | *string* | MD5 checksum of the object | | `objInfo.ContentType` | *string* | Content type of the object | | `objInfo.Size` | *int64* | Size of the object | **Example** ```go objInfo, err := minioClient.StatObject(context.Background(), "mybucket", "myobject", minio.StatObjectOptions{}) if err != nil { fmt.Println(err) return } fmt.Println(objInfo) ``` ### RemoveObject(ctx context.Context, bucketName, objectName string, opts minio.RemoveObjectOptions) error Removes an object with some specified options **Parameters** | Param | Type | Description | |:-------------|:----------------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `opts` | *minio.RemoveObjectOptions* | Allows user to set options | **minio.RemoveObjectOptions** | Field | Type | Description | |:------------------------|:------------------------------|:--------------------------------------------------------------------------------------------------------------------------------| | `opts.GovernanceBypass` | *bool* | Set the bypass governance header to delete an object locked with GOVERNANCE mode | | `opts.VersionID` | *string* | Version ID of the object to delete | | `opts.Internal` | *minio.AdvancedRemoveOptions* | This option is intended for internal use by MinIO server and should not be set unless the application is aware of intended use. | ```go opts := minio.RemoveObjectOptions{ GovernanceBypass: true, VersionID: "myversionid", } err = minioClient.RemoveObject(context.Background(), "mybucket", "myobject", opts) if err != nil { fmt.Println(err) return } ``` ### PutObjectRetention(ctx context.Context, bucketName, objectName string, opts minio.PutObjectRetentionOptions) error Applies object retention lock onto an object. **Parameters** | Param | Type | Description | |:-------------|:----------------------------------|:---------------------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `opts` | *minio.PutObjectRetentionOptions* | Allows user to set options like retention mode, expiry date and version id | ### RemoveObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, opts RemoveObjectsOptions) <-chan RemoveObjectError Removes a list of objects obtained from an input channel. The call sends a delete request to the server up to 1000 objects at a time. The errors observed are sent over the error channel. Parameters | Param | Type | Description | |:-------------|:-----------------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectsCh` | *chan minio.ObjectInfo* | Channel of objects to be removed | | `opts` | *minio.RemoveObjectsOptions* | Allows user to set options | **minio.RemoveObjectsOptions** | Field | Type | Description | |:------------------------|:-------|:---------------------------------------------------------------------------------| | `opts.GovernanceBypass` | *bool* | Set the bypass governance header to delete an object locked with GOVERNANCE mode | **Return Values** | Param | Type | Description | |:----------|:---------------------------------|:---------------------------------------------------------| | `errorCh` | *<-chan minio.RemoveObjectError* | Receive-only channel of errors observed during deletion. | ```go objectsCh := make(chan minio.ObjectInfo) // Send object names that are needed to be removed to objectsCh go func() { defer close(objectsCh) // List all objects from a bucket-name with a matching prefix. for object := range minioClient.ListObjects(context.Background(), "my-bucketname", "my-prefixname", true, nil) { if object.Err != nil { log.Fatalln(object.Err) } objectsCh <- object } }() opts := minio.RemoveObjectsOptions{ GovernanceBypass: true, } for rErr := range minioClient.RemoveObjects(context.Background(), "my-bucketname", objectsCh, opts) { fmt.Println("Error detected during deletion: ", rErr) } ``` ### RemoveObjectsWithResult(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, opts RemoveObjectsOptions) <-chan RemoveObjectResult Removes a list of objects and returns both successful deletions and errors. This is an enhanced version of RemoveObjects that provides complete deletion results. **Parameters** | Param | Type | Description | |:-------------|:-----------------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectsCh` | *chan minio.ObjectInfo* | Channel of objects to be removed | | `opts` | *minio.RemoveObjectsOptions* | Allows user to set options | **Return Values** | Param | Type | Description | |:-----------|:----------------------------------|:-------------------------------------------------------| | `resultCh` | *<-chan minio.RemoveObjectResult* | Channel of results including both successes and errors | **minio.RemoveObjectResult** | Field | Type | Description | |:-------------|:---------|:------------------------------------------| | `ObjectName` | *string* | Name of the object | | `VersionID` | *string* | Version ID of the object (if versioned) | | `Err` | *error* | Error during deletion (nil if successful) | **Example** ```go objectsCh := make(chan minio.ObjectInfo) go func() { defer close(objectsCh) for object := range minioClient.ListObjects(context.Background(), "my-bucketname", minio.ListObjectsOptions{Prefix: "my-prefixname", Recursive: true}) { if object.Err != nil { log.Fatalln(object.Err) } objectsCh <- object } }() opts := minio.RemoveObjectsOptions{ GovernanceBypass: true, } successCount := 0 errorCount := 0 for result := range minioClient.RemoveObjectsWithResult(context.Background(), "my-bucketname", objectsCh, opts) { if result.Err != nil { fmt.Printf("Error deleting %s: %v\n", result.ObjectName, result.Err) errorCount++ } else { fmt.Printf("Successfully deleted %s\n", result.ObjectName) successCount++ } } fmt.Printf("Deleted: %d, Errors: %d\n", successCount, errorCount) ``` ### RemoveObjectsWithIter(ctx context.Context, bucketName string, objectsIter iter.Seq[ObjectInfo], opts RemoveObjectsOptions) (iter.Seq[RemoveObjectResult], error) Iterator-based version of RemoveObjects for Go 1.23+. Removes objects using an iterator input and returns results via an iterator. **Parameters** | Param | Type | Description | |:--------------|:-----------------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectsIter` | *iter.Seq[minio.ObjectInfo]* | Iterator of objects to be removed | | `opts` | *minio.RemoveObjectsOptions* | Allows user to set options | **Return Values** | Param | Type | Description | |:-------------|:-------------------------------------|:-------------------------------------| | `resultIter` | *iter.Seq[minio.RemoveObjectResult]* | Iterator yielding removal results | | `err` | *error* | Error initializing removal operation | **Example** ```go opts := minio.ListObjectsOptions{ Prefix: "my-prefixname", Recursive: true, } removeOpts := minio.RemoveObjectsOptions{ GovernanceBypass: true, } for err := range minioClient.RemoveObjectsWithIter(context.Background(), "my-bucketname", minioClient.ListObjectsIter(context.Background(), "my-bucketname", opts), removeOpts) { fmt.Printf("Error detected during deletion: %v\n", err) } ``` ### GetObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) Returns retention set on a given object. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `versionID` | *string* | Version ID of the object | ```go mode, retainUntilDate, err := minioClient.GetObjectRetention(context.Background(), "mybucket", "myobject", "") if err != nil { fmt.Println(err) return } fmt.Printf("Retention mode: %v, Retain until: %v\n", mode, retainUntilDate) ``` ### PutObjectLegalHold(ctx context.Context, bucketName, objectName string, opts minio.PutObjectLegalHoldOptions) error Applies legal-hold onto an object. **Parameters** | Param | Type | Description | |:-------------|:----------------------------------|:------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `opts` | *minio.PutObjectLegalHoldOptions* | Allows user to set options like status and version id | *minio.PutObjectLegalHoldOptions* | Field | Type | Description | |:-----------------|:--------------------------|:-----------------------------------------------| | `opts.Status` | \**minio.LegalHoldStatus* | Legal-Hold status to be set | | `opts.VersionID` | *string* | Version ID of the object to apply retention on | ```go s := minio.LegalHoldEnabled opts := minio.PutObjectLegalHoldOptions{ Status: &s, } err = minioClient.PutObjectLegalHold(context.Background(), "mybucket", "myobject", opts) if err != nil { fmt.Println(err) return } ``` ### GetObjectLegalHold(ctx context.Context, bucketName, objectName, versionID string) (status *LegalHoldStatus, err error) Returns legal-hold status on a given object. **Parameters** | Param | Type | Description | |:-------------|:----------------------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `opts` | *minio.GetObjectLegalHoldOptions* | Allows user to set options like version id | ```go opts := minio.GetObjectLegalHoldOptions{} err = minioClient.GetObjectLegalHold(context.Background(), "mybucket", "myobject", opts) if err != nil { fmt.Println(err) return } ``` ### SelectObjectContent(ctx context.Context, bucketName string, objectsName string, expression string, options SelectObjectOptions) *SelectResults Parameters | Param | Type | Description | |:-------------|:----------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `ctx` | *context.Context* | Request context | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `options` | *SelectObjectOptions* | Query Options | **Return Values** | Param | Type | Description | |:----------------|:----------------|:------------------------------------------------------------------------------------------------| | `SelectResults` | *SelectResults* | Is an io.ReadCloser object which can be directly passed to csv.NewReader for processing output. | ```go // Initialize minio client object. minioClient, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), Secure: useSSL, }) opts := minio.SelectObjectOptions{ Expression: "select count(*) from s3object", ExpressionType: minio.QueryExpressionTypeSQL, InputSerialization: minio.SelectObjectInputSerialization{ CompressionType: minio.SelectCompressionNONE, CSV: &minio.CSVInputOptions{ FileHeaderInfo: minio.CSVFileHeaderInfoNone, RecordDelimiter: "\n", FieldDelimiter: ",", }, }, OutputSerialization: minio.SelectObjectOutputSerialization{ CSV: &minio.CSVOutputOptions{ RecordDelimiter: "\n", FieldDelimiter: ",", }, }, } reader, err := s3Client.SelectObjectContent(context.Background(), "mycsvbucket", "mycsv.csv", opts) if err != nil { log.Fatalln(err) } defer reader.Close() if _, err := io.Copy(os.Stdout, reader); err != nil { log.Fatalln(err) } ``` ### PutObjectTagging(ctx context.Context, bucketName, objectName string, otags *tags.Tags, opts PutObjectTaggingOptions) error set new object Tags to the given object, replaces/overwrites any existing tags. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `objectTags` | \**tags.Tags* | Map with Object Tag's Key and Value | **Example** ```go err = minioClient.PutObjectTagging(context.Background(), bucketName, objectName, objectTags) if err != nil { fmt.Println(err) return } ``` ### GetObjectTagging(ctx context.Context, bucketName, objectName string) (*tags.Tags, error) Fetch Object Tags from the given object **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | **Example** ```go tags, err = minioClient.GetObjectTagging(context.Background(), bucketName, objectName) if err != nil { fmt.Println(err) return } fmt.Printf("Fetched Tags: %s", tags) ``` ### RemoveObjectTagging(ctx context.Context, bucketName, objectName string) error Remove Object Tags from the given object **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | **Example** ```go err = minioClient.RemoveObjectTagging(context.Background(), bucketName, objectName) if err != nil { fmt.Println(err) return } ``` ### RestoreObject(ctx context.Context, bucketName, objectName, versionID string, opts minio.RestoreRequest) error Restore or perform SQL operations on an archived object **Parameters** | Param | Type | Description | |:-------------|:----------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `versionID` | *string* | Version ID of the object | | `opts` | _minio.RestoreRequest | Restore request options | **Example** ```go opts := minio.RestoreRequest{} opts.SetDays(1) opts.SetGlacierJobParameters(minio.GlacierJobParameters{Tier: minio.TierStandard}) err = s3Client.RestoreObject(context.Background(), "your-bucket", "your-object", "", opts) if err != nil { log.Fatalln(err) } ``` ### GetObjectAttributes(ctx context.Context, bucketName, objectName string, opts ObjectAttributesOptions) (*ObjectAttributes, error) Returns a stream of the object data. Most of the common errors occur when reading the stream. **Parameters** | Param | Type | Description | |:-------------|:--------------------------------|:----------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `opts` | *minio.ObjectAttributesOptions* | Configuration for pagination and selection of object attributes | **minio.ObjectAttributesOptions** | Field | Type | Description | |:----------------------------|:---------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------| | `opts.ServerSideEncryption` | *encrypt.ServerSide* | Interface provided by `encrypt` package to specify server-side-encryption. (For more information see https://godoc.org/github.com/minio/minio-go/v7\) | | `opts.MaxParts` | _int | This option defines how many parts should be returned by the API | | `opts.VersionID` | _string | VersionID defines which version of the object will be used | | `opts.PartNumberMarker` | _int | This options defines which part number pagination will start after, the part which number is equal to PartNumberMarker will not be included in the response | **Return Value** | Param | Type | Description | |:-------------------|:---------------------------|:-----------------------------------------------------------------------------------| | `objectAttributes` | \**minio.ObjectAttributes* | *minio.ObjectAttributes* contains the information about the object and it's parts. | **Example** ```go objectAttributes, err := c.GetObjectAttributes( context.Background(), "your-bucket", "your-object", minio.ObjectAttributesOptions{ VersionID: "object-version-id", NextPartMarker: 0, MaxParts: 100, }) if err != nil { fmt.Println(err) return } fmt.Println(objectAttributes) ``` ### RemoveIncompleteUpload(ctx context.Context, bucketName, objectName string) error Removes a partially uploaded object. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | **Example** ```go err = minioClient.RemoveIncompleteUpload(context.Background(), "mybucket", "myobject") if err != nil { fmt.Println(err) return } ``` 1. Presigned operations ----------------------- ### PresignedGetObject(ctx context.Context, bucketName, objectName string, expiry time.Duration, reqParams url.Values) (*url.URL, error) Generates a presigned URL for HTTP GET operations. Browsers/Mobile clients may point to this URL to directly download objects even if the bucket is private. This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The maximum expiry is 604800 seconds (i.e. 7 days) and minimum is 1 second. **Parameters** | Param | Type | Description | |:-------------|:------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `expiry` | *time.Duration* | Expiry of presigned URL in seconds | | `reqParams` | *url.Values* | Additional response header overrides supports *response-expires*, *response-content-type*, *response-cache-control*, *response-content-disposition*. | **Example** ```go // Set request parameters for content-disposition. reqParams := make(url.Values) reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") // Generates a presigned url which expires in a day. presignedURL, err := minioClient.PresignedGetObject(context.Background(), "mybucket", "myobject", time.Second*24*60*60, reqParams) if err != nil { fmt.Println(err) return } fmt.Println("Successfully generated presigned URL", presignedURL) ``` ### PresignedPutObject(ctx context.Context, bucketName, objectName string, expiry time.Duration) (*url.URL, error) Generates a presigned URL for HTTP PUT operations. Browsers/Mobile clients may point to this URL to upload objects directly to a bucket even if it is private. This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The default expiry is set to 7 days. NOTE: you can upload to S3 only with specified object name. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `expiry` | *time.Duration* | Expiry of presigned URL in seconds | **Example** ```go // Generates a url which expires in a day. expiry := time.Second * 24 * 60 * 60 // 1 day. presignedURL, err := minioClient.PresignedPutObject(context.Background(), "mybucket", "myobject", expiry) if err != nil { fmt.Println(err) return } fmt.Println("Successfully generated presigned URL", presignedURL) ``` ### PresignedHeadObject(ctx context.Context, bucketName, objectName string, expiry time.Duration, reqParams url.Values) (*url.URL, error) Generates a presigned URL for HTTP HEAD operations. Browsers/Mobile clients may point to this URL to directly get metadata from objects even if the bucket is private. This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The default expiry is set to 7 days. **Parameters** | Param | Type | Description | |:-------------|:------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `expiry` | *time.Duration* | Expiry of presigned URL in seconds | | `reqParams` | *url.Values* | Additional response header overrides supports *response-expires*, *response-content-type*, *response-cache-control*, *response-content-disposition*. | **Example** ```go // Set request parameters for content-disposition. reqParams := make(url.Values) reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") // Generates a presigned url which expires in a day. presignedURL, err := minioClient.PresignedHeadObject(context.Background(), "mybucket", "myobject", time.Second*24*60*60, reqParams) if err != nil { fmt.Println(err) return } fmt.Println("Successfully generated presigned URL", presignedURL) ``` ### PresignedPostPolicy(ctx context.Context, post PostPolicy) (*url.URL, map[string]string, error) Allows setting policy conditions to a presigned URL for POST operations. Policies such as bucket name to receive object uploads, key name prefixes, expiry policy may be set. ```go // Initialize policy condition config. policy := minio.NewPostPolicy() // Apply upload policy restrictions: policy.SetBucket("mybucket") policy.SetKey("myobject") policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days // Only allow 'png' images. policy.SetContentType("image/png") // Only allow content size in range 1KB to 1MB. policy.SetContentLengthRange(1024, 1024*1024) // Add a user metadata using the key "custom" and value "user" policy.SetUserMetadata("custom", "user") // Get the POST form key/value object: url, formData, err := minioClient.PresignedPostPolicy(context.Background(), policy) if err != nil { fmt.Println(err) return } // POST your content from the command line using `curl` fmt.Printf("curl ") for k, v := range formData { fmt.Printf("-F %s=%s ", k, v) } fmt.Printf("-F file=@/etc/bash.bashrc ") fmt.Printf("%s\n", url) ``` 1. Bucket policy/notification operations ---------------------------------------- ### SetBucketPolicy(ctx context.Context, bucketname, policy string) error Set access permissions on bucket or an object prefix. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `policy` | *string* | Policy to be set | **Return Values** | Param | Type | Description | |:------|:--------|:---------------| | `err` | *error* | Standard Error | **Example** ```go policy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::my-bucketname/*"],"Sid": ""}]}` err = minioClient.SetBucketPolicy(context.Background(), "my-bucketname", policy) if err != nil { fmt.Println(err) return } ``` ### GetBucketPolicy(ctx context.Context, bucketName string) (policy string, error) Get access permissions on a bucket or a prefix. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:---------|:---------|:--------------------------------| | `policy` | *string* | Policy returned from the server | | `err` | *error* | Standard Error | **Example** ```go policy, err := minioClient.GetBucketPolicy(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } ``` ### GetBucketNotification(ctx context.Context, bucketName string) (notification.Configuration, error) Get notification configuration on a bucket. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:---------|:-----------------------------|:------------------------------------------------------| | `config` | *notification.Configuration* | structure which holds all notification configurations | | `err` | *error* | Standard Error | **Example** ```go bucketNotification, err := minioClient.GetBucketNotification(context.Background(), "mybucket") if err != nil { fmt.Println("Failed to get bucket notification configurations for mybucket", err) return } for _, queueConfig := range bucketNotification.QueueConfigs { for _, e := range queueConfig.Events { fmt.Println(e + " event is enabled") } } ``` ### SetBucketNotification(ctx context.Context, bucketName string, config notification.Configuration) error Set a new bucket notification on a bucket. **Parameters** | Param | Type | Description | |:-------------|:-----------------------------|:------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `config` | *notification.Configuration* | Represents the XML to be sent to the configured web service | **Return Values** | Param | Type | Description | |:------|:--------|:---------------| | `err` | *error* | Standard Error | **Example** ```go queueArn := notification.NewArn("aws", "sqs", "us-east-1", "804605494417", "PhotoUpdate") queueConfig := notification.NewConfig(queueArn) queueConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) queueConfig.AddFilterPrefix("photos/") queueConfig.AddFilterSuffix(".jpg") config := notification.Configuration{} config.AddQueue(queueConfig) err = minioClient.SetBucketNotification(context.Background(), "mybucket", config) if err != nil { fmt.Println("Unable to set the bucket notification: ", err) return } ``` ### RemoveAllBucketNotification(ctx context.Context, bucketName string) error Remove all configured bucket notifications on a bucket. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:------|:--------|:---------------| | `err` | *error* | Standard Error | **Example** ```go err = minioClient.RemoveAllBucketNotification(context.Background(), "mybucket") if err != nil { fmt.Println("Unable to remove bucket notifications.", err) return } ``` ### ListenBucketNotification(context context.Context, bucketName, prefix, suffix string, events []string) <-chan notification.Info ListenBucketNotification API receives bucket notification events through the notification channel. The returned notification channel has two fields 'Records' and 'Err'. - 'Records' holds the notifications received from the server. - 'Err' indicates any error while processing the received notifications. NOTE: Notification channel is closed at the first occurrence of an error. **Parameters** | Param | Type | Description | |:-------------|:-----------|:-----------------------------------------------| | `bucketName` | *string* | Bucket to listen notifications on | | `prefix` | *string* | Object key prefix to filter notifications for | | `suffix` | *string* | Object key suffix to filter notifications for | | `events` | *[]string* | Enables notifications for specific event types | **Return Values** | Param | Type | Description | |:-------------------|:-------------------------|:--------------------------------| | `notificationInfo` | *chan notification.Info* | Channel of bucket notifications | **minio.NotificationInfo** |Field |Type |Description | |`notificationInfo.Records` | *[]notification.Event* | Collection of notification events | |`notificationInfo.Err` | *error* | Carries any error occurred during the operation (Standard Error) | **Example** ```go // Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. for notificationInfo := range minioClient.ListenBucketNotification(context.Background(), "mybucket", "myprefix/", ".mysuffix", []string{ "s3:ObjectCreated:*", "s3:ObjectAccessed:*", "s3:ObjectRemoved:*", }) { if notificationInfo.Err != nil { fmt.Println(notificationInfo.Err) } fmt.Println(notificationInfo) } ``` ### ListenNotification(context context.Context, prefix, suffix string, events []string) <-chan notification.Info ListenNotification API receives bucket and object notification events through the notification channel. The returned notification channel has two fields 'Records' and 'Err'. - 'Records' holds the notifications received from the server. - 'Err' indicates any error while processing the received notifications. NOTE: Notification channel is closed at the first occurrence of an error. **Parameters** | Param | Type | Description | |:-------------|:-----------|:-----------------------------------------------| | `bucketName` | *string* | Bucket to listen notifications on | | `prefix` | *string* | Object key prefix to filter notifications for | | `suffix` | *string* | Object key suffix to filter notifications for | | `events` | *[]string* | Enables notifications for specific event types | **Return Values** | Param | Type | Description | |:-------------------|:-------------------------|:-----------------------------------| | `notificationInfo` | *chan notification.Info* | Read channel for all notifications | **minio.NotificationInfo** |Field |Type |Description | |`notificationInfo.Records` | *[]notification.Event* | Collection of notification events | |`notificationInfo.Err` | *error* | Carries any error occurred during the operation (Standard Error) | **Example** ```go // Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. for notificationInfo := range minioClient.ListenNotification(context.Background(), "myprefix/", ".mysuffix", []string{ "s3:BucketCreated:*", "s3:BucketRemoved:*", "s3:ObjectCreated:*", "s3:ObjectAccessed:*", "s3:ObjectRemoved:*", }) { if notificationInfo.Err != nil { fmt.Println(notificationInfo.Err) } fmt.Println(notificationInfo) } ``` ### SetBucketLifecycle(ctx context.Context, bucketName string, config *lifecycle.Configuration) error Set lifecycle on bucket or an object prefix. **Parameters** | Param | Type | Description | |:-------------|:--------------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `config` | *lifecycle.Configuration* | Lifecycle to be set | **Return Values** | Param | Type | Description | |:------|:--------|:---------------| | `err` | *error* | Standard Error | **Example** ```go config := lifecycle.NewConfiguration() config.Rules = []lifecycle.Rule{ { ID: "expire-bucket", Status: "Enabled", Expiration: lifecycle.Expiration{ Days: 365, }, }, } err = minioClient.SetBucketLifecycle(context.Background(), "my-bucketname", config) if err != nil { fmt.Println(err) return } ``` ### GetBucketLifecycle(ctx context.Context, bucketName string) (*lifecycle.Configuration, error) Get lifecycle on a bucket or a prefix. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:---------|:--------------------------|:-----------------------------------| | `config` | *lifecycle.Configuration* | Lifecycle returned from the server | | `err` | *error* | Standard Error | **Example** ```go lifecycle, err := minioClient.GetBucketLifecycle(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } ``` ### SetBucketEncryption(ctx context.Context, bucketName string, config *sse.Configuration) error Set default encryption configuration on a bucket. **Parameters** | Param | Type | Description | |:-------------|:--------------------|:----------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `config` | *sse.Configuration* | Structure that holds default encryption configuration to be set | **Return Values** | Param | Type | Description | |:------|:--------|:---------------| | `err` | *error* | Standard Error | **Example** ```go s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // Set default encryption configuration on an S3 bucket err = s3Client.SetBucketEncryption(context.Background(), "my-bucketname", sse.NewConfigurationSSES3()) if err != nil { log.Fatalln(err) } ``` ### GetBucketEncryption(ctx context.Context, bucketName string) (*sse.Configuration, error) Get default encryption configuration set on a bucket. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:---------|:--------------------|:------------------------------------------------------| | `config` | *sse.Configuration* | Structure that holds default encryption configuration | | `err` | *error* | Standard Error | **Example** ```go s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // Get default encryption configuration set on an S3 bucket and print it out encryptionConfig, err := s3Client.GetBucketEncryption(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } fmt.Printf("%+v\n", encryptionConfig) ``` ### RemoveBucketEncryption(ctx context.Context, bucketName string) error Remove default encryption configuration set on a bucket. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:------|:--------|:---------------| | `err` | *error* | Standard Error | **Example** ```go err := s3Client.RemoveBucketEncryption(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } // "my-bucket" is successfully deleted/removed. ``` ### SetObjectLockConfig(ctx context.Context, bucketName string, mode *RetentionMode, validity *uint, unit *ValidityUnit) error Set object lock configuration in given bucket. mode, validity and unit are either all set or all nil. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `mode` | *RetentionMode* | Retention mode to be set | | `validity` | *uint* | Validity period to be set | | `unit` | *ValidityUnit* | Unit of validity period | **Return Values** | Param | Type | Description | |:------|:--------|:---------------| | `err` | *error* | Standard Error | **Example** ```go mode := Governance validity := uint(30) unit := Days err = minioClient.SetObjectLockConfig(context.Background(), "my-bucketname", &mode, &validity, &unit) if err != nil { fmt.Println(err) return } ``` ### GetObjectLockConfig(ctx context.Context, bucketName string) (objectLock string, mode *RetentionMode, validity *uint, unit *ValidityUnit, err error) Get object lock configuration of given bucket. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:-------------|:----------------|:------------------------| | `objectLock` | *objectLock* | lock enabled status | | `mode` | *RetentionMode* | Current retention mode | | `validity` | *uint* | Current validity period | | `unit` | *ValidityUnit* | Unit of validity period | | `err` | *error* | Standard Error | **Example** ```go enabled, mode, validity, unit, err := minioClient.GetObjectLockConfig(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } fmt.Println("object lock is %s for this bucket", enabled) if mode != nil { fmt.Printf("%v mode is enabled for %v %v for bucket 'my-bucketname'\n", *mode, *validity, *unit) } else { fmt.Println("No mode is enabled for bucket 'my-bucketname'") } ``` ### EnableVersioning(ctx context.Context, bucketName string) error Enable bucket versioning support. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:------|:--------|:---------------| | `err` | *error* | Standard Error | **Example** ```go err := minioClient.EnableVersioning(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } fmt.Println("versioning enabled for bucket 'my-bucketname'") ``` ### SuspendVersioning(ctx context.Context, bucketName string) error Suspend bucket versioning support. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:------|:--------|:---------------| | `err` | *error* | Standard Error | **Example** ```go err := minioClient.SuspendVersioning(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } fmt.Println("versioning suspended for bucket 'my-bucketname'") ``` ### GetBucketVersioning(ctx context.Context, bucketName string) (BucketVersioningConfiguration, error) Get versioning configuration set on a bucket. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:----------------|:--------------------------------------|:----------------------------------------------| | `configuration` | *minio.BucketVersioningConfiguration* | Structure that holds versioning configuration | | `err` | *error* | Standard Error | **Example** ```go s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // Get versioning configuration set on an S3 bucket and print it out versioningConfig, err := s3Client.GetBucketVersioning(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } fmt.Printf("%+v\n", versioningConfig) ``` ### SetBucketReplication(ctx context.Context, bucketName string, cfg replication.Config) error Set replication configuration on a bucket. Role can be obtained by first defining the replication target on MinIO using `mc admin bucket remote set` to associate the source and destination buckets for replication with the replication endpoint. **Parameters** | Param | Type | Description | |:-------------|:---------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `cfg` | *replication.Config* | Replication configuration to be set | **Return Values** | Param | Type | Description | |:------|:--------|:---------------| | `err` | *error* | Standard Error | **Example** ```go replicationStr := ` Disabled string string string string string ... string string string string string integer string ` replicationConfig := replication.Config{} if err := xml.Unmarshal([]byte(replicationStr), &replicationConfig); err != nil { log.Fatalln(err) } replicationConfig.Role = "arn:minio:s3::598361bf-3cec-49a7-b529-ce870a34d759:*" err = minioClient.SetBucketReplication(context.Background(), "my-bucketname", replicationConfig) if err != nil { fmt.Println(err) return } ``` ### GetBucketReplication(ctx context.Context, bucketName string) (replication.Config, error) Get current replication config on a bucket. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:--------------|:---------------------|:--------------------------------------------| | `replication` | *replication.Config* | Replication config returned from the server | | `err` | *error* | Standard Error | **Example** ```go replication, err := minioClient.GetBucketReplication(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } ``` ### RemoveBucketReplication(ctx context.Context, bucketName string) error Removes replication configuration on a bucket. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:------|:--------|:---------------| | `err` | *error* | Standard Error | **Example** ```go err = minioClient.RemoveBucketReplication(context.Background(), "my-bucketname") if err != nil { fmt.Println(err) return } ``` ### CancelBucketReplicationResync(ctx context.Context, bucketName string, tgtArn string) (id string, err error) Cancels in progress replication resync (MinIO AiStor Only API) **Parameters** | Param | Type | Description | |:-------------|:------------------|:---------------------------------------------------| | `ctx` | *context.Context* | Custom context of timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `tgtArn` | *string* | Target Amazon Resource Name | **Return Values** |Param |Type |Description | |:---|:--|:---| |`id`|*string*| Recieved upon successful cancellation of replication resync| |`err`| *error*| Standard Error| **Example** ```go id, err := minioClient.CancelBucketReplicationResync(context.Background(), "my-bucket-name", "my-target-arn") if err != nil { fmt.Println(err) return } ``` ### GetBucketReplicationMetrics(ctx context.Context, bucketName string) (replication.Metrics, error) Get latest replication metrics on a bucket. This is a MinIO specific extension. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:----------|:----------------------|:---------------------------------------------| | `metrics` | *replication.Metrics* | Replication metrics returned from the server | | `err` | *error* | Standard Error | **Example** ```go replMetrics, err := minioClient.GetBucketReplicationMetrics(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } ``` ### GetBucketReplicationMetricsV2(ctx context.Context, bucketName string) (replication.MetricsV2, error) Get latest replication metrics using the V2 API on a bucket. This is a MinIO specific extension with enhanced metrics. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:----------|:------------------------|:------------------------------------------------------| | `metrics` | *replication.MetricsV2* | Enhanced replication metrics returned from the server | | `err` | *error* | Standard Error | **Example** ```go replMetrics, err := minioClient.GetBucketReplicationMetricsV2(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } fmt.Printf("Replication metrics: %+v\n", replMetrics) ``` ### CheckBucketReplication(ctx context.Context, bucketName string) error Validate whether replication is properly configured for a bucket. This is a MinIO specific extension. **Parameters** | Param | Type | Description | |:-------------|:------------------|:----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |:------|:--------|:-------------------------------------------------------------| | `err` | *error* | Returns nil if valid, or error describing validation failure | **Example** ```go err := minioClient.CheckBucketReplication(context.Background(), "my-bucketname") if err != nil { log.Printf("Replication configuration is invalid: %v\n", err) } else { log.Println("Replication configuration is valid") } ``` ### ResetBucketReplication(ctx context.Context, bucketName string, olderThan time.Duration) (string, error) Initiate replication of previously replicated objects. Requires ExistingObjectReplication to be enabled in the replication configuration. This is a MinIO specific extension. **Parameters** | Param | Type | Description | |:-------------|:------------------|:--------------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `olderThan` | *time.Duration* | Only replicate objects older than this duration (0 for all objects) | **Return Values** | Param | Type | Description | |:----------|:---------|:------------------------------------| | `resetID` | *string* | Reset ID for tracking the operation | | `err` | *error* | Standard Error | **Example** ```go // Reset replication for all objects resetID, err := minioClient.ResetBucketReplication(context.Background(), "my-bucketname", 0) if err != nil { log.Fatalln(err) } fmt.Printf("Replication reset initiated with ID: %s\n", resetID) ``` ### ResetBucketReplicationOnTarget(ctx context.Context, bucketName string, olderThan time.Duration, tgtArn string) (replication.ResyncTargetsInfo, error) Initiate replication of previously replicated objects to a specific target. Requires ExistingObjectReplication to be enabled in the replication configuration. This is a MinIO specific extension. **Parameters** | Param | Type | Description | |:-------------|:------------------|:--------------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `olderThan` | *time.Duration* | Only replicate objects older than this duration (0 for all objects) | | `tgtArn` | *string* | ARN of the target to reset replication for | **Return Values** | Param | Type | Description | |:-------------|:--------------------------------|:--------------------------| | `resyncInfo` | *replication.ResyncTargetsInfo* | Resync target information | | `err` | *error* | Standard Error | **Example** ```go resyncInfo, err := minioClient.ResetBucketReplicationOnTarget(context.Background(), "my-bucketname", 0, "arn:aws:s3:::target-bucket") if err != nil { log.Fatalln(err) } fmt.Printf("Resync info: %+v\n", resyncInfo) ``` ### GetBucketReplicationResyncStatus(ctx context.Context, bucketName, arn string) (replication.ResyncTargetsInfo, error) Retrieve the status of a replication resync operation. This is a MinIO specific extension. **Parameters** | Param | Type | Description | |:-------------|:------------------|:-------------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `arn` | *string* | ARN of the replication target (empty string for all targets) | **Return Values** | Param | Type | Description | |:-------------|:--------------------------------|:--------------------------| | `resyncInfo` | *replication.ResyncTargetsInfo* | Resync status information | | `err` | *error* | Standard Error | **Example** ```go // Get resync status for all targets resyncInfo, err := minioClient.GetBucketReplicationResyncStatus(context.Background(), "my-bucketname", "") if err != nil { log.Fatalln(err) } fmt.Printf("Resync status: %+v\n", resyncInfo) ``` 1. Client custom settings ------------------------- ### SetAppInfo(appName, appVersion string) Add custom application details to User-Agent. **Parameters** | Param | Type | Description | |--------------|----------|---------------------------------------------------------| | `appName` | *string* | Name of the application performing the API requests. | | `appVersion` | *string* | Version of the application performing the API requests. | **Example** ```go // Set Application name and version to be used in subsequent API requests. minioClient.SetAppInfo("myCloudApp", "1.0.0") ``` ### TraceOn(outputStream io.Writer) Enables HTTP tracing. The trace is written to the io.Writer provided. If outputStream is nil, trace is written to os.Stdout. **Parameters** | Param | Type | Description | |----------------|-------------|------------------------------------------| | `outputStream` | *io.Writer* | HTTP trace is written into outputStream. | ### TraceOff() Disables HTTP tracing. ### SetS3TransferAccelerate(acceleratedEndpoint string) Set AWS S3 transfer acceleration endpoint for all API requests hereafter. NOTE: This API applies only to AWS S3 and is a no operation for S3 compatible object storage services. **Parameters** | Param | Type | Description | |-----------------------|----------|-----------------------------------------------| | `acceleratedEndpoint` | *string* | Set to new S3 transfer acceleration endpoint. | ### EndpointURL() *url.URL Returns the URL of the S3-compatible endpoint that this client connects to. Returns a copy to prevent modification of internal state. **Return Values** | Param | Type | Description | |------------|-------------|--------------| | `endpoint` | \**url.URL* | Endpoint URL | **Example** ```go endpointURL := minioClient.EndpointURL() fmt.Printf("Connected to: %s\n", endpointURL.String()) ``` ### GetCreds() (credentials.Value, error) Returns the current credentials being used by the client. Useful for debugging or when credentials need to be passed to other components. **Return Values** | Param | Type | Description | |---------|---------------------|------------------------------| | `creds` | *credentials.Value* | Current credentials | | `err` | *error* | Error retrieving credentials | **Example** ```go creds, err := minioClient.GetCreds() if err != nil { log.Fatalln(err) } fmt.Printf("Access Key: %s\n", creds.AccessKeyID) ``` 1. Additional Operations ------------------------ ### SetBucketCors(ctx context.Context, bucketName string, corsConfig *cors.Config) error Set CORS (Cross-Origin Resource Sharing) configuration on a bucket. **Parameters** | Param | Type | Description | |--------------|-------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `corsConfig` | \**cors.Config* | CORS configuration to be set | **Example** {% raw %} ```go // Create CORS configuration corsConfig := &cors.Config{ CORSRules: []cors.Rule{{ AllowedHeaders: []string{"*"}, AllowedMethods: []string{"PUT", "GET", "DELETE"}, AllowedOrigins: []string{"*"}, MaxAgeSeconds: 3000, }}, } err := minioClient.SetBucketCors(context.Background(), "mybucket", corsConfig) if err != nil { log.Fatalln(err) } ``` {% endraw %} ### GetBucketCors(ctx context.Context, bucketName string) (*cors.Config, error) Get CORS configuration of a bucket. **Parameters** | Param | Type | Description | |--------------|-------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |--------------|-----------------|--------------------| | `corsConfig` | \**cors.Config* | CORS configuration | | `err` | *error* | Standard Error | **Example** ```go corsConfig, err := minioClient.GetBucketCors(context.Background(), "mybucket") if err != nil { log.Fatalln(err) } fmt.Printf("CORS configuration: %+v\n", corsConfig) ``` ### GetBucketQOS(ctx context.Context, bucket string) (*QOSConfig, error) Get Quality of Service (QoS) configuration for a bucket. This is a MinIO-specific API. **Parameters** | Param | Type | Description | |----------|-------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucket` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |-------------|---------------|------------------------------| | `qosConfig` | \**QOSConfig* | QoS configuration with rules | | `err` | *error* | Standard Error | **Example** ```go qosConfig, err := minioClient.GetBucketQOS(context.Background(), "mybucket") if err != nil { log.Fatalln(err) } fmt.Printf("QoS configuration: %+v\n", qosConfig) ``` ### SetBucketQOS(ctx context.Context, bucket string, qosCfg *QOSConfig) error Set Quality of Service (QoS) configuration for a bucket. This is a MinIO-specific API. **Parameters** | Param | Type | Description | |----------|-------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucket` | *string* | Name of the bucket | | `qosCfg` | \**QOSConfig* | QoS configuration to apply | **Example** ```go qosConfig := &minio.QOSConfig{ Version: "v1", Rules: []minio.QOSRule{{ ID: "rule1", Priority: 1, ObjectPrefix: "logs/", API: "s3:GetObject", Rate: 1000, Burst: 100, Limit: "rps", }}, } err := minioClient.SetBucketQOS(context.Background(), "mybucket", qosConfig) if err != nil { log.Fatalln(err) } ``` ### GetBucketQOSMetrics(ctx context.Context, bucketName, nodeName string) ([]QOSNodeStats, error) Get Quality of Service (QoS) metrics for a bucket. This is a MinIO-specific API. **Parameters** | Param | Type | Description | |--------------|-------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `nodeName` | *string* | Name of the node (empty string for all nodes) | **Return Values** | Param | Type | Description | |-----------|------------------|----------------------| | `metrics` | *[]QOSNodeStats* | QoS metrics per node | | `err` | *error* | Standard Error | **Example** ```go // Get QoS metrics for all nodes metrics, err := minioClient.GetBucketQOSMetrics(context.Background(), "mybucket", "") if err != nil { log.Fatalln(err) } for _, nodeStats := range metrics { fmt.Printf("Node: %s, Stats: %+v\n", nodeStats.NodeName, nodeStats.Stats) } ``` ### GenerateInventoryConfigYAML(ctx context.Context, bucket, id string) (string, error) Generate a YAML template for an inventory configuration. This is a MinIO-specific API. **Parameters** | Param | Type | Description | |----------|-------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucket` | *string* | Name of the bucket | | `id` | *string* | Unique identifier for the inventory configuration | **Return Values** | Param | Type | Description | |----------------|----------|----------------------| | `yamlTemplate` | *string* | YAML template string | | `err` | *error* | Standard Error | **Example** ```go yamlTemplate, err := minioClient.GenerateInventoryConfigYAML(context.Background(), "mybucket", "inventory1") if err != nil { log.Fatalln(err) } fmt.Println(yamlTemplate) ``` ### PutBucketInventoryConfiguration(ctx context.Context, bucket, id, yamlDef string, opts ...InventoryPutConfigOption) error Create or update an inventory configuration for a bucket. This is a MinIO-specific API. **Parameters** | Param | Type | Description | |-----------|-------------------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucket` | *string* | Name of the bucket | | `id` | *string* | Unique identifier for the inventory configuration | | `yamlDef` | *string* | YAML definition of the inventory configuration | | `opts` | *...InventoryPutConfigOption* | Optional configuration options | **Example** ```go yamlDef := `--- version: v1 destination: bucket: destination-bucket prefix: inventory-reports schedule: frequency: Daily ...` err := minioClient.PutBucketInventoryConfiguration(context.Background(), "mybucket", "inventory1", yamlDef) if err != nil { log.Fatalln(err) } ``` ### GetBucketInventoryConfiguration(ctx context.Context, bucket, id string) (*InventoryConfiguration, error) Retrieve the inventory configuration for a bucket. This is a MinIO-specific API. **Parameters** | Param | Type | Description | |----------|-------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucket` | *string* | Name of the bucket | | `id` | *string* | Unique identifier for the inventory configuration | **Return Values** | Param | Type | Description | |----------|----------------------------|-------------------------| | `config` | \**InventoryConfiguration* | Inventory configuration | | `err` | *error* | Standard Error | **Example** ```go config, err := minioClient.GetBucketInventoryConfiguration(context.Background(), "mybucket", "inventory1") if err != nil { log.Fatalln(err) } fmt.Printf("Inventory: %+v\n", config) ``` ### DeleteBucketInventoryConfiguration(ctx context.Context, bucket, id string) error Delete an inventory configuration from a bucket. This is a MinIO-specific API. **Parameters** | Param | Type | Description | |----------|-------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucket` | *string* | Name of the bucket | | `id` | *string* | Unique identifier for the inventory configuration | **Example** ```go err := minioClient.DeleteBucketInventoryConfiguration(context.Background(), "mybucket", "inventory1") if err != nil { log.Fatalln(err) } ``` ### ListBucketInventoryConfigurations(ctx context.Context, bucket, continuationToken string) (*InventoryListResult, error) List up to 100 inventory configurations for a bucket with pagination support. This is a MinIO-specific API. **Parameters** | Param | Type | Description | |---------------------|-------------------|-------------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucket` | *string* | Name of the bucket | | `continuationToken` | *string* | Token for pagination (empty string for first request) | **Return Values** | Param | Type | Description | |----------|-------------------------|--------------------------------------------------------| | `result` | \**InventoryListResult* | List result with configurations and continuation token | | `err` | *error* | Standard Error | **Example** ```go result, err := minioClient.ListBucketInventoryConfigurations(context.Background(), "mybucket", "") if err != nil { log.Fatalln(err) } for _, item := range result.Items { fmt.Printf("Inventory ID: %s\n", item.ID) } if result.NextContinuationToken != "" { // Fetch next page nextResult, _ := minioClient.ListBucketInventoryConfigurations(context.Background(), "mybucket", result.NextContinuationToken) } ``` ### ListBucketInventoryConfigurationsIterator(ctx context.Context, bucket string) iter.Seq2[InventoryConfiguration, error] Return an iterator that lists all inventory configurations for a bucket. This is a MinIO-specific API. Requires Go 1.23+. **Parameters** | Param | Type | Description | |----------|-------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucket` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |------------|--------------------------------------------|---------------------------------------------| | `iterator` | *iter.Seq2[InventoryConfiguration, error]* | Iterator yielding configurations and errors | **Example** ```go for config, err := range minioClient.ListBucketInventoryConfigurationsIterator(context.Background(), "mybucket") { if err != nil { log.Fatalln(err) break } fmt.Printf("Inventory ID: %s, Bucket: %s\n", config.ID, config.Bucket) } ``` ### GetBucketInventoryJobStatus(ctx context.Context, bucket, id string) (*InventoryJobStatus, error) Retrieve the status of an inventory job for a bucket. This is a MinIO-specific API. **Parameters** | Param | Type | Description | |----------|-------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucket` | *string* | Name of the bucket | | `id` | *string* | Unique identifier for the inventory job | **Return Values** | Param | Type | Description | |----------|------------------------|--------------------------------------------------| | `status` | \**InventoryJobStatus* | Job status including state, progress, and errors | | `err` | *error* | Standard Error | **InventoryJobStatus Fields** The `InventoryJobStatus` struct contains comprehensive job information: | Field | Type | Description | |----------------------|-----------------|----------------------------------------------------------------------| | `Bucket` | string | Source bucket name | | `ID` | string | Inventory configuration ID | | `User` | string | User who created the job | | `AccessKey` | string | Access key used for job execution | | `Schedule` | string | Job schedule (once, hourly, daily, weekly, monthly, yearly) | | `State` | string | Current job state (sleeping, pending, running, completed, etc.) | | `NextScheduledTime` | time.Time | When next execution will start (periodic jobs only) | | `StartTime` | time.Time | When current/last execution started | | `EndTime` | time.Time | When execution completed | | `LastUpdate` | time.Time | Last time job metadata was updated | | `Scanned` | string | Last scanned object path | | `Matched` | string | Last matched object path | | `ScannedCount` | uint64 | Total objects scanned | | `MatchedCount` | uint64 | Total objects matched by filters | | `RecordsWritten` | uint64 | Number of records written to output files | | `OutputFilesCount` | uint64 | Number of output files created | | `ExecutionTime` | time.Duration | Total execution time | | `NumStarts` | uint64 | Number of times job has started | | `NumErrors` | uint64 | Total errors encountered | | `NumLockLosses` | uint64 | Number of distributed lock losses | | `ManifestPath` | string | Full path to manifest.json file | | `RetryAttempts` | uint64 | Number of retry attempts | | `LastFailTime` | time.Time | When last failure occurred (only present on errors) | | `LastFailErrors` | []string | Up to 5 most recent error messages (only present on errors) | **Example** ```go status, err := minioClient.GetBucketInventoryJobStatus(context.Background(), "mybucket", "inventory1") if err != nil { log.Fatalln(err) } fmt.Printf("Job ID: %s, State: %s\n", status.ID, status.State) fmt.Printf("Progress: %d/%d objects scanned/matched\n", status.ScannedCount, status.MatchedCount) fmt.Printf("Output: %d records in %d files\n", status.RecordsWritten, status.OutputFilesCount) if !status.StartTime.IsZero() { fmt.Printf("Started: %s\n", status.StartTime) if status.ExecutionTime > 0 { fmt.Printf("Execution time: %s\n", status.ExecutionTime) } } if status.ManifestPath != "" { fmt.Printf("Manifest: %s\n", status.ManifestPath) } if len(status.LastFailErrors) > 0 { fmt.Printf("Recent errors: %v\n", status.LastFailErrors) } ``` ### PutObjectsSnowball(ctx context.Context, bucketName string, opts SnowballOptions, objs <-chan SnowballObject) error Bulk upload multiple objects using snowball archive method for efficient batch operations. **Parameters** | Param | Type | Description | |--------------|-------------------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `opts` | *minio.SnowballOptions* | Snowball upload options | | `objs` | *<-chan minio.SnowballObject* | Channel of objects to upload | ### PromptObject(ctx context.Context, bucketName, objectName, prompt string, opts PromptObjectOptions) (io.ReadCloser, error) Perform language model inference with prompt and object context for AI/ML integration. **Parameters** | Param | Type | Description | |--------------|-----------------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | | `objectName` | *string* | Name of the object | | `prompt` | *string* | AI prompt text | | `opts` | *minio.PromptObjectOptions* | Prompt operation options | **Return Values** | Param | Type | Description | |----------|-----------------|--------------------| | `result` | *io.ReadCloser* | AI response stream | | `err` | *error* | Standard Error | ### GetBucketLocation(ctx context.Context, bucketName string) (string, error) Get the region/location constraint of a bucket. **Parameters** | Param | Type | Description | |--------------|-------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |------------|----------|------------------------| | `location` | *string* | Bucket region/location | | `err` | *error* | Standard Error | **Example** ```go location, err := minioClient.GetBucketLocation(context.Background(), "mybucket") if err != nil { log.Fatalln(err) } fmt.Printf("Bucket location: %s\n", location) ``` ### GetBucketReplicationMetrics(ctx context.Context, bucketName string) (replication.Metrics, error) Get replication metrics for a bucket. This is a MinIO specific extension. **Parameters** | Param | Type | Description | |--------------|-------------------|-----------------------------------------------------| | `ctx` | *context.Context* | Custom context for timeout/cancellation of the call | | `bucketName` | *string* | Name of the bucket | **Return Values** | Param | Type | Description | |-----------|-----------------------|---------------------| | `metrics` | *replication.Metrics* | Replication metrics | | `err` | *error* | Standard Error | **Example** ```go metrics, err := minioClient.GetBucketReplicationMetrics(context.Background(), "mybucket") if err != nil { log.Fatalln(err) } fmt.Printf("Replication metrics: %+v\n", metrics) ``` ### TraceErrorsOnlyOn(outputStream io.Writer) Enable HTTP tracing for errors only. The trace is written to the io.Writer provided. **Parameters** | Param | Type | Description | |----------------|-------------|------------------------------------------------| | `outputStream` | *io.Writer* | HTTP error trace is written into outputStream. | ### TraceErrorsOnlyOff() Disable HTTP error-only tracing. ### SetS3EnableDualstack(enabled bool) Enable or disable S3 dual-stack endpoints which support both IPv4 and IPv6. **Parameters** | Param | Type | Description | |-----------|--------|---------------------------------------------| | `enabled` | *bool* | Enable dual-stack if true, disable if false | ### IsOnline() bool Check if the MinIO client is online and can reach the server. **Return Value** | Param | Type | Description | |----------|--------|--------------------------| | `online` | *bool* | true if client is online | ### IsOffline() bool Check if the MinIO client is offline and cannot reach the server. **Return Value** | Param | Type | Description | |-----------|--------|---------------------------| | `offline` | *bool* | true if client is offline | ### HealthCheck(hcDuration time.Duration) (context.CancelFunc, error) Start continuous health check monitoring of the MinIO server. **Parameters** | Param | Type | Description | |--------------|-----------------|-----------------------| | `hcDuration` | *time.Duration* | Health check interval | **Return Values** | Param | Type | Description | |----------|----------------------|---------------------------------| | `cancel` | *context.CancelFunc* | Function to cancel health check | | `err` | *error* | Standard Error | minio-go-7.0.97/endpoints.go000066400000000000000000000163111510244170000156620ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2024 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio type awsS3Endpoint struct { endpoint string dualstackEndpoint string } type awsS3ExpressEndpoint struct { regionalEndpoint string zonalEndpoints []string } var awsS3ExpressEndpointMap = map[string]awsS3ExpressEndpoint{ "us-east-1": { "s3express-control.us-east-1.amazonaws.com", []string{ "s3express-use1-az4.us-east-1.amazonaws.com", "s3express-use1-az5.us-east-1.amazonaws.com", "3express-use1-az6.us-east-1.amazonaws.com", }, }, "us-east-2": { "s3express-control.us-east-2.amazonaws.com", []string{ "s3express-use2-az1.us-east-2.amazonaws.com", "s3express-use2-az2.us-east-2.amazonaws.com", }, }, "us-west-2": { "s3express-control.us-west-2.amazonaws.com", []string{ "s3express-usw2-az1.us-west-2.amazonaws.com", "s3express-usw2-az3.us-west-2.amazonaws.com", "s3express-usw2-az4.us-west-2.amazonaws.com", }, }, "ap-south-1": { "s3express-control.ap-south-1.amazonaws.com", []string{ "s3express-aps1-az1.ap-south-1.amazonaws.com", "s3express-aps1-az3.ap-south-1.amazonaws.com", }, }, "ap-northeast-1": { "s3express-control.ap-northeast-1.amazonaws.com", []string{ "s3express-apne1-az1.ap-northeast-1.amazonaws.com", "s3express-apne1-az4.ap-northeast-1.amazonaws.com", }, }, "eu-west-1": { "s3express-control.eu-west-1.amazonaws.com", []string{ "s3express-euw1-az1.eu-west-1.amazonaws.com", "s3express-euw1-az3.eu-west-1.amazonaws.com", }, }, "eu-north-1": { "s3express-control.eu-north-1.amazonaws.com", []string{ "s3express-eun1-az1.eu-north-1.amazonaws.com", "s3express-eun1-az2.eu-north-1.amazonaws.com", "s3express-eun1-az3.eu-north-1.amazonaws.com", }, }, } // awsS3EndpointMap Amazon S3 endpoint map. var awsS3EndpointMap = map[string]awsS3Endpoint{ "us-east-1": { "s3.us-east-1.amazonaws.com", "s3.dualstack.us-east-1.amazonaws.com", }, "us-east-2": { "s3.us-east-2.amazonaws.com", "s3.dualstack.us-east-2.amazonaws.com", }, "us-iso-east-1": { "s3.us-iso-east-1.c2s.ic.gov", "s3.dualstack.us-iso-east-1.c2s.ic.gov", }, "us-isob-east-1": { "s3.us-isob-east-1.sc2s.sgov.gov", "s3.dualstack.us-isob-east-1.sc2s.sgov.gov", }, "us-iso-west-1": { "s3.us-iso-west-1.c2s.ic.gov", "s3.dualstack.us-iso-west-1.c2s.ic.gov", }, "us-west-2": { "s3.us-west-2.amazonaws.com", "s3.dualstack.us-west-2.amazonaws.com", }, "us-west-1": { "s3.us-west-1.amazonaws.com", "s3.dualstack.us-west-1.amazonaws.com", }, "ca-central-1": { "s3.ca-central-1.amazonaws.com", "s3.dualstack.ca-central-1.amazonaws.com", }, "ca-west-1": { "s3.ca-west-1.amazonaws.com", "s3.dualstack.ca-west-1.amazonaws.com", }, "eu-west-1": { "s3.eu-west-1.amazonaws.com", "s3.dualstack.eu-west-1.amazonaws.com", }, "eu-west-2": { "s3.eu-west-2.amazonaws.com", "s3.dualstack.eu-west-2.amazonaws.com", }, "eu-west-3": { "s3.eu-west-3.amazonaws.com", "s3.dualstack.eu-west-3.amazonaws.com", }, "eu-central-1": { "s3.eu-central-1.amazonaws.com", "s3.dualstack.eu-central-1.amazonaws.com", }, "eu-central-2": { "s3.eu-central-2.amazonaws.com", "s3.dualstack.eu-central-2.amazonaws.com", }, "eu-north-1": { "s3.eu-north-1.amazonaws.com", "s3.dualstack.eu-north-1.amazonaws.com", }, "eu-south-1": { "s3.eu-south-1.amazonaws.com", "s3.dualstack.eu-south-1.amazonaws.com", }, "eu-south-2": { "s3.eu-south-2.amazonaws.com", "s3.dualstack.eu-south-2.amazonaws.com", }, "ap-east-1": { "s3.ap-east-1.amazonaws.com", "s3.dualstack.ap-east-1.amazonaws.com", }, "ap-south-1": { "s3.ap-south-1.amazonaws.com", "s3.dualstack.ap-south-1.amazonaws.com", }, "ap-south-2": { "s3.ap-south-2.amazonaws.com", "s3.dualstack.ap-south-2.amazonaws.com", }, "ap-southeast-1": { "s3.ap-southeast-1.amazonaws.com", "s3.dualstack.ap-southeast-1.amazonaws.com", }, "ap-southeast-2": { "s3.ap-southeast-2.amazonaws.com", "s3.dualstack.ap-southeast-2.amazonaws.com", }, "ap-southeast-3": { "s3.ap-southeast-3.amazonaws.com", "s3.dualstack.ap-southeast-3.amazonaws.com", }, "ap-southeast-4": { "s3.ap-southeast-4.amazonaws.com", "s3.dualstack.ap-southeast-4.amazonaws.com", }, "ap-northeast-1": { "s3.ap-northeast-1.amazonaws.com", "s3.dualstack.ap-northeast-1.amazonaws.com", }, "ap-northeast-2": { "s3.ap-northeast-2.amazonaws.com", "s3.dualstack.ap-northeast-2.amazonaws.com", }, "ap-northeast-3": { "s3.ap-northeast-3.amazonaws.com", "s3.dualstack.ap-northeast-3.amazonaws.com", }, "af-south-1": { "s3.af-south-1.amazonaws.com", "s3.dualstack.af-south-1.amazonaws.com", }, "me-central-1": { "s3.me-central-1.amazonaws.com", "s3.dualstack.me-central-1.amazonaws.com", }, "me-south-1": { "s3.me-south-1.amazonaws.com", "s3.dualstack.me-south-1.amazonaws.com", }, "sa-east-1": { "s3.sa-east-1.amazonaws.com", "s3.dualstack.sa-east-1.amazonaws.com", }, "us-gov-west-1": { "s3.us-gov-west-1.amazonaws.com", "s3.dualstack.us-gov-west-1.amazonaws.com", }, "us-gov-east-1": { "s3.us-gov-east-1.amazonaws.com", "s3.dualstack.us-gov-east-1.amazonaws.com", }, "cn-north-1": { "s3.cn-north-1.amazonaws.com.cn", "s3.dualstack.cn-north-1.amazonaws.com.cn", }, "cn-northwest-1": { "s3.cn-northwest-1.amazonaws.com.cn", "s3.dualstack.cn-northwest-1.amazonaws.com.cn", }, "il-central-1": { "s3.il-central-1.amazonaws.com", "s3.dualstack.il-central-1.amazonaws.com", }, "ap-southeast-5": { "s3.ap-southeast-5.amazonaws.com", "s3.dualstack.ap-southeast-5.amazonaws.com", }, "ap-southeast-7": { "s3.ap-southeast-7.amazonaws.com", "s3.dualstack.ap-southeast-7.amazonaws.com", }, "mx-central-1": { "s3.mx-central-1.amazonaws.com", "s3.dualstack.mx-central-1.amazonaws.com", }, "ap-east-2": { "s3.ap-east-2.amazonaws.com", "s3.dualstack.ap-east-2.amazonaws.com", }, } // getS3ExpressEndpoint get Amazon S3 Express endpoing based on the region // optionally if zonal is set returns first zonal endpoint. func getS3ExpressEndpoint(region string, zonal bool) (endpoint string) { s3ExpEndpoint, ok := awsS3ExpressEndpointMap[region] if !ok { return "" } if zonal { return s3ExpEndpoint.zonalEndpoints[0] } return s3ExpEndpoint.regionalEndpoint } // getS3Endpoint get Amazon S3 endpoint based on the bucket location. func getS3Endpoint(bucketLocation string, useDualstack bool) (endpoint string) { s3Endpoint, ok := awsS3EndpointMap[bucketLocation] if !ok { // Default to 's3.us-east-1.amazonaws.com' endpoint. if useDualstack { return "s3.dualstack.us-east-1.amazonaws.com" } return "s3.us-east-1.amazonaws.com" } if useDualstack { return s3Endpoint.dualstackEndpoint } return s3Endpoint.endpoint } minio-go-7.0.97/examples/000077500000000000000000000000001510244170000151445ustar00rootroot00000000000000minio-go-7.0.97/examples/minio/000077500000000000000000000000001510244170000162575ustar00rootroot00000000000000minio-go-7.0.97/examples/minio/checkbucketreplication.go000066400000000000000000000032671510244170000233230ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("play.min.io", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // Get bucket replication configuration from S3 err = s3Client.CheckBucketReplication(context.Background(), "bucket") if err != nil { log.Fatalln(err) } log.Println("Bucket replication configuration is valid") } minio-go-7.0.97/examples/minio/getbucketreplicationmetrics.go000066400000000000000000000033041510244170000244040ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "log" "os" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("play.min.io", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } s3Client.TraceOn(os.Stderr) // Get replication metrics for a bucket m, err := s3Client.GetBucketReplicationMetrics(context.Background(), "bucket") if err != nil { log.Fatalln(err) } fmt.Println("Replication metrics for my-bucketname:", m) } minio-go-7.0.97/examples/minio/go.mod000066400000000000000000000021141510244170000173630ustar00rootroot00000000000000module github.com/minio/minio-go/examples/minio go 1.23.0 toolchain go1.24.1 // Overridden by `replace` below, to point all versions at the local minio-go source, so version shouldn't matter here. require github.com/minio/minio-go/v7 v7.0.73 require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.11 // indirect github.com/klauspost/crc32 v1.3.0 // indirect github.com/minio/crc64nvme v1.1.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rs/xid v1.6.0 // indirect github.com/tinylib/msgp v1.3.0 // indirect golang.org/x/crypto v0.39.0 // indirect golang.org/x/net v0.41.0 // indirect golang.org/x/sys v0.34.0 // indirect golang.org/x/text v0.26.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/minio/minio-go/v7 => ../.. minio-go-7.0.97/examples/minio/go.sum000066400000000000000000000064251510244170000174210ustar00rootroot00000000000000github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q= github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= minio-go-7.0.97/examples/minio/listen-notification.go000066400000000000000000000036411510244170000225740ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. minioClient, err := minio.New("play.min.io", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. for notificationInfo := range minioClient.ListenNotification(context.Background(), "PREFIX", "SUFFIX", []string{ "s3:BucketCreated:*", "s3:BucketRemoved:*", "s3:ObjectCreated:*", "s3:ObjectAccessed:*", "s3:ObjectRemoved:*", }) { if notificationInfo.Err != nil { log.Fatalln(notificationInfo.Err) } log.Println(notificationInfo) } } minio-go-7.0.97/examples/minio/listenbucketnotification.go000066400000000000000000000036061510244170000237160ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. minioClient, err := minio.New("play.min.io", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. for notificationInfo := range minioClient.ListenBucketNotification(context.Background(), "YOUR-BUCKET", "PREFIX", "SUFFIX", []string{ "s3:ObjectCreated:*", "s3:ObjectAccessed:*", "s3:ObjectRemoved:*", }) { if notificationInfo.Err != nil { log.Fatalln(notificationInfo.Err) } log.Println(notificationInfo) } } minio-go-7.0.97/examples/minio/listobjectsV2WithMetadata.go000066400000000000000000000035071510244170000236450ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("play.min.io", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { fmt.Println(err) return } opts := minio.ListObjectsOptions{ WithMetadata: true, Prefix: "my-prefixname", Recursive: true, } // List all objects from a bucket-name with a matching prefix. for object := range s3Client.ListObjects(context.Background(), "my-bucketname", opts) { if object.Err != nil { fmt.Println(object.Err) return } fmt.Println(object) } return } minio-go-7.0.97/examples/minio/put-object-fan-out.go000066400000000000000000000051671510244170000222420ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2023 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "io" "log" "os" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" ) func main() { const ( // Note: These constants are dummy values, // please replace them with values for your setup. YOURACCESSKEYID = "Q3AM3UQ867SPQQA43P2F" YOURSECRETACCESSKEY = "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" YOURENDPOINT = "play.min.io" YOURBUCKET = "mybucket" // 'mc mb play/mybucket' if it does not exist. ) // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. minioClient, err := minio.New(YOURENDPOINT, &minio.Options{ Creds: credentials.NewStaticV4(YOURACCESSKEYID, YOURSECRETACCESSKEY, ""), Secure: true, }) if err != nil { log.Fatalln(err) } // Enable tracing. minioClient.TraceOn(os.Stdout) filePath := "my-testfile" // Specify a local file that we will upload // Open a local file that we will upload file, err := os.Open(filePath) if err != nil { log.Fatalln(err) } defer file.Close() cs, err := minio.ChecksumCRC32C.ChecksumReader(file) if err != nil { log.Fatalln(err) } // Seek to beginning before upload. file.Seek(0, io.SeekStart) fanOutReq := minio.PutObjectFanOutRequest{ Entries: []minio.PutObjectFanOutEntry{ {Key: "my1-prefix/1.txt"}, {Key: "my1-prefix/2.txt"}, {Key: "my1-prefix/3.txt"}, {Key: "my1-prefix/4.txt"}, {Key: "my1-prefix/5.txt"}, {Key: "my1-prefix/6.txt"}, }, SSE: encrypt.NewSSE(), Checksum: cs, } fanOutResp, err := minioClient.PutObjectFanOut(context.Background(), YOURBUCKET, file, fanOutReq) if err != nil { log.Fatalln(err) } for _, resp := range fanOutResp { fmt.Println(resp) } } minio-go-7.0.97/examples/minio/putobjectsnowball.go000066400000000000000000000063651510244170000223610ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "bytes" "context" "fmt" "log" "math/rand" "os" "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { const ( // Note: These constants are dummy values, // please replace them with values for your setup. YOURACCESSKEYID = "Q3AM3UQ867SPQQA43P2F" YOURSECRETACCESSKEY = "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" YOURENDPOINT = "play.min.io" YOURBUCKET = "mybucket" // 'mc mb play/mybucket' if it does not exist. ) // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. minioClient, err := minio.New(YOURENDPOINT, &minio.Options{ Creds: credentials.NewStaticV4(YOURACCESSKEYID, YOURSECRETACCESSKEY, ""), Secure: true, }) if err != nil { log.Fatalln(err) } minioClient.TraceOn(os.Stdout) input := make(chan minio.SnowballObject, 1) opts := minio.SnowballOptions{ Opts: minio.PutObjectOptions{}, // Keep in memory. We use this since we have small total payload. InMemory: true, // Compress data when uploading to a MinIO host. Compress: true, } // Generate a shared prefix. rng := rand.New(rand.NewSource(time.Now().UnixNano())) prefix := []byte("aaaaaaaaaaaaaaa") for i := range prefix { prefix[i] += byte(rng.Intn(25)) } // Generate go func() { defer close(input) // Create 100 objects for i := 0; i < 100; i++ { // With random size 0 -> 10000 size := rng.Intn(10000) key := fmt.Sprintf("%s/%d-%d.bin", string(prefix), i, size) input <- minio.SnowballObject{ // Create path to store objects within the bucket. Key: key, Size: int64(size), ModTime: time.Now(), Content: bytes.NewBuffer(make([]byte, size)), Close: func() { fmt.Println(key, "Close function called") }, } } }() // Collect and upload all entries. err = minioClient.PutObjectsSnowball(context.TODO(), YOURBUCKET, opts, input) if err != nil { log.Fatalln(err) } // Objects successfully uploaded. // List the content of the prefix: lopts := minio.ListObjectsOptions{ Recursive: true, Prefix: string(prefix) + "/", } // List all objects from a bucket-name with a matching prefix. for object := range minioClient.ListObjects(context.Background(), YOURBUCKET, lopts) { if object.Err != nil { log.Fatalln(object.Err) } fmt.Println(object.Key, "Size:", object.Size) } } minio-go-7.0.97/examples/s3/000077500000000000000000000000001510244170000154715ustar00rootroot00000000000000minio-go-7.0.97/examples/s3/bucketexists.go000066400000000000000000000032301510244170000205330ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } found, err := s3Client.BucketExists(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } if found { log.Println("Bucket found.") } else { log.Println("Bucket not found.") } } minio-go-7.0.97/examples/s3/composeobject.go000066400000000000000000000053271510244170000206630ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // Enable trace. // s3Client.TraceOn(os.Stderr) // Prepare source decryption key (here we assume same key to // decrypt all source objects.) decKey, _ := encrypt.NewSSEC([]byte{1, 2, 3}) // Source objects to concatenate. We also specify decryption // key for each src1 := minio.CopySrcOptions{ Bucket: "bucket1", Object: "object1", Encryption: decKey, MatchETag: "31624deb84149d2f8ef9c385918b653a", } src2 := minio.CopySrcOptions{ Bucket: "bucket2", Object: "object2", Encryption: decKey, MatchETag: "f8ef9c385918b653a31624deb84149d2", } src3 := minio.CopySrcOptions{ Bucket: "bucket3", Object: "object3", Encryption: decKey, MatchETag: "5918b653a31624deb84149d2f8ef9c38", } // Create slice of sources. srcs := []minio.CopySrcOptions{src1, src2, src3} // Prepare destination encryption key encKey, _ := encrypt.NewSSEC([]byte{8, 9, 0}) // Create destination info dst := minio.CopyDestOptions{ Bucket: "bucket", Object: "object", Encryption: encKey, } uploadInfo, err := s3Client.ComposeObject(context.Background(), dst, srcs...) if err != nil { log.Fatalln(err) } log.Println("Composed object successfully:", uploadInfo) } minio-go-7.0.97/examples/s3/copyobject-with-new-tags.go000066400000000000000000000044251510244170000226620ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // Enable trace. // s3Client.TraceOn(os.Stderr) // Source object src := minio.CopySrcOptions{ Bucket: "my-sourcebucketname", Object: "my-sourceobjectname", // All following conditions are allowed and can be combined together. // Set modified condition, copy object modified since 2014 April. MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), } // Destination object dst := minio.CopyDestOptions{ Bucket: "my-bucketname", Object: "my-objectname", ReplaceTags: true, UserTags: map[string]string{ "Tag1": "Value1", "Tag2": "Value2", }, } // Initiate copy object. ui, err := s3Client.CopyObject(context.Background(), dst, src) if err != nil { log.Fatalln(err) } log.Printf("Copied %s, successfully to %s - UploadInfo %v\n", dst, src, ui) } minio-go-7.0.97/examples/s3/copyobject.go000066400000000000000000000051071510244170000201640ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // Enable trace. // s3Client.TraceOn(os.Stderr) // Source object src := minio.CopySrcOptions{ Bucket: "my-sourcebucketname", Object: "my-sourceobjectname", // All following conditions are allowed and can be combined together. // Set modified condition, copy object modified since 2014 April. MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), // Set unmodified condition, copy object unmodified since 2014 April. // MatchUnmodifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), // Set matching ETag condition, copy object which matches the following ETag. // MatchETag: "31624deb84149d2f8ef9c385918b653a", // Set matching ETag copy object which does not match the following ETag. // NoMatchETag: "31624deb84149d2f8ef9c385918b653a", } // Destination object dst := minio.CopyDestOptions{ Bucket: "my-bucketname", Object: "my-objectname", } // Initiate copy object. ui, err := s3Client.CopyObject(context.Background(), dst, src) if err != nil { log.Fatalln(err) } log.Printf("Copied %s, successfully to %s - UploadInfo %v\n", dst, src, ui) } minio-go-7.0.97/examples/s3/enableversioning.go000066400000000000000000000031211510244170000213470ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } err = s3Client.EnableVersioning(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } log.Println("Enabled") } minio-go-7.0.97/examples/s3/fgetobject.go000066400000000000000000000033041510244170000201340ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname // and my-filename.csv are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } if err := s3Client.FGetObject(context.Background(), "my-bucketname", "my-objectname", "my-filename.csv", minio.GetObjectOptions{}); err != nil { log.Fatalln(err) } log.Println("Successfully saved my-filename.csv") } minio-go-7.0.97/examples/s3/fputencrypted-object.go000066400000000000000000000045531510244170000221670ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } filePath := "my-testfile" // Specify a local file that we will upload bucketname := "my-bucketname" // Specify a bucket name - the bucket must already exist objectName := "my-objectname" // Specify a object name password := "correct horse battery staple" // Specify your password. DO NOT USE THIS ONE - USE YOUR OWN. // New SSE-C where the cryptographic key is derived from a password and the objectname + bucketname as salt encryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketname+objectName)) // Encrypt file content and upload to the server uploadedInfo, err := s3Client.FPutObject(context.Background(), bucketname, objectName, filePath, minio.PutObjectOptions{ServerSideEncryption: encryption}) if err != nil { log.Fatalln(err) } log.Println("Uploaded", "my-objectname:", uploadedInfo) } minio-go-7.0.97/examples/s3/fputobject.go000066400000000000000000000033561510244170000201740ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname // and my-filename.csv are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } if _, err := s3Client.FPutObject(context.Background(), "my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ ContentType: "application/csv", }); err != nil { log.Fatalln(err) } log.Println("Successfully uploaded my-filename.csv") } minio-go-7.0.97/examples/s3/get-encrypted-object.go000066400000000000000000000047331510244170000220450ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "io" "log" "os" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and // my-testfile are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } bucketname := "my-bucketname" // Specify a bucket name - the bucket must already exist objectName := "my-objectname" // Specify a object name - the object must already exist password := "correct horse battery staple" // Specify your password. DO NOT USE THIS ONE - USE YOUR OWN. // New SSE-C where the cryptographic key is derived from a password and the objectname + bucketname as salt encryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketname+objectName)) // Get the encrypted object reader, err := s3Client.GetObject(context.Background(), bucketname, objectName, minio.GetObjectOptions{ServerSideEncryption: encryption}) if err != nil { log.Fatalln(err) } defer reader.Close() // Local file which holds plain data localFile, err := os.Create("my-testfile") if err != nil { log.Fatalln(err) } defer localFile.Close() if _, err := io.Copy(localFile, reader); err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/getbucketencryption.go000066400000000000000000000034061510244170000221130ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // Get default encryption configuration set on an S3 bucket, // and print out the encryption configuration. encryptionConfig, err := s3Client.GetBucketEncryption(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } fmt.Printf("%+v\n", encryptionConfig) } minio-go-7.0.97/examples/s3/getbucketlifecycle.go000066400000000000000000000037251510244170000216640ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "encoding/xml" "log" "os" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // Get bucket lifecycle from S3 lifecycle, err := s3Client.GetBucketLifecycle(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } // Save the lifecycle document to a file localLifecycleFile, err := os.Create("lifecycle.json") if err != nil { log.Fatalln(err) } defer localLifecycleFile.Close() enc := xml.NewEncoder(localLifecycleFile) enc.Indent(" ", " ") if err := enc.Encode(lifecycle); err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/getbucketnotification.go000066400000000000000000000035031510244170000224050ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) notifications, err := s3Client.GetBucketNotification(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } log.Println("Bucket notification are successfully retrieved.") for _, topicConfig := range notifications.TopicConfigs { for _, e := range topicConfig.Events { log.Println(e + " event is enabled.") } } } minio-go-7.0.97/examples/s3/getbucketpolicy.go000066400000000000000000000035741510244170000212260ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "io" "log" "os" "strings" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) policy, err := s3Client.GetBucketPolicy(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } // Create policy file localFile, err := os.Create("policy.json") if err != nil { log.Fatalln(err) } defer localFile.Close() policyReader := strings.NewReader(policy) if _, err := io.Copy(localFile, policyReader); err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/getbucketreplication.go000066400000000000000000000037401510244170000222330ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "encoding/json" "log" "os" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // Get bucket replication configuration from S3 replicationCfg, err := s3Client.GetBucketReplication(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } // Create replication config file localReplicationCfgFile, err := os.Create("replication.xml") if err != nil { log.Fatalln(err) } defer localReplicationCfgFile.Close() replBytes, err := json.Marshal(replicationCfg) if err != nil { log.Fatalln(err) } localReplicationCfgFile.Write(replBytes) } minio-go-7.0.97/examples/s3/getbuckettagging.go000066400000000000000000000032201510244170000213330ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "log" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } tags, err := s3Client.GetBucketTagging(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } fmt.Printf("Fetched Object Tags: %s", tags) } minio-go-7.0.97/examples/s3/getbucketversioning.go000066400000000000000000000033761510244170000221120ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // Get versioning configuration set on an S3 bucket, // and print out the versioning configuration. versioningConfig, err := s3Client.GetBucketVersioning(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } fmt.Printf("%+v\n", versioningConfig) } minio-go-7.0.97/examples/s3/getobject-client-encryption.go000066400000000000000000000043471510244170000234420ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "os" "path" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/sio" "golang.org/x/crypto/argon2" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } obj, err := s3Client.GetObject(context.Background(), "my-bucketname", "my-objectname", minio.GetObjectOptions{}) if err != nil { log.Fatalln(err) } localFile, err := os.Create("my-testfile") if err != nil { log.Fatalln(err) } defer localFile.Close() password := []byte("myfavoritepassword") // Change as per your needs. salt := []byte(path.Join("my-bucketname", "my-objectname")) // Change as per your needs. _, err = sio.Decrypt(localFile, obj, sio.Config{ Key: argon2.IDKey(password, salt, 1, 64*1024, 4, 32), // generate a 256 bit long key. }) if err != nil { log.Fatalln(err) } log.Println("Successfully decrypted 'my-objectname' to local file 'my-testfile'") } minio-go-7.0.97/examples/s3/getobject-override-respheaders.go000066400000000000000000000047571510244170000241230ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package s3 import ( "context" "io" "log" "os" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and // my-testfile are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // response-cache-control=ResponseCacheControl // response-content-disposition=ResponseContentDisposition // response-content-encoding=ResponseContentEncoding // response-content-language=ResponseContentLanguage // response-content-type=ResponseContentType // response-expires=ResponseExpires opt := minio.GetObjectOptions{} opt.SetReqParam("response-content-disposition", `attachment; filename="testing.txt"`) opt.SetReqParam("response-cache-control", "No-cache") opt.SetReqParam("response-content-encoding", "x-gzip") opt.SetReqParam("response-expires", "Mon, 02 Jan 2006 15:04:05 GMT") reader, err := s3Client.GetObject(context.Background(), "my-bucketname", "my-objectname", opt) if err != nil { log.Fatalln(err) } defer reader.Close() localFile, err := os.Create("my-testfile") if err != nil { log.Fatalln(err) } defer localFile.Close() stat, err := reader.Stat() if err != nil { log.Fatalln(err) } if _, err := io.CopyN(localFile, reader, stat.Size); err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/getobject.go000066400000000000000000000036561510244170000200000ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "io" "log" "os" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and // my-testfile are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } reader, err := s3Client.GetObject(context.Background(), "my-bucketname", "my-objectname", minio.GetObjectOptions{}) if err != nil { log.Fatalln(err) } defer reader.Close() localFile, err := os.Create("my-testfile") if err != nil { log.Fatalln(err) } defer localFile.Close() stat, err := reader.Stat() if err != nil { log.Fatalln(err) } if _, err := io.CopyN(localFile, reader, stat.Size); err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/getobjectacl.go000066400000000000000000000041711510244170000204510ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2018-2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "log" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and // my-testfile are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) objectInfo, err := s3Client.GetObjectACL(context.Background(), "my-bucketname", "my-objectname") if err != nil { log.Fatalln(err) } // print object owner informations fmt.Printf(`Object owner: Display name: %q ID: %q `, objectInfo.Owner.DisplayName, objectInfo.Owner.ID) // print object grant informations for _, g := range objectInfo.Grant { fmt.Printf(`Object grant: - Display name: %q - ID: %q - URI: %q - Permission: %q `, g.Grantee.DisplayName, g.Grantee.ID, g.Grantee.URI, g.Permission) } // print all value header (acl, metadata, standard header value...) for k, v := range objectInfo.Metadata { fmt.Println("key:", k) fmt.Printf(" - value: %v\n", v) } } minio-go-7.0.97/examples/s3/getobjectlegalhold.go000066400000000000000000000034051510244170000216440ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and // my-testfile are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } opts := minio.GetObjectLegalHoldOptions{} lh, err := s3Client.GetObjectLegalHold(context.Background(), "my-bucket", "my-object", opts) if err != nil { log.Fatalln(err) } fmt.Printf("Legal Hold on object is %s", lh) log.Println("Get object legal-hold on my-object successfully.") } minio-go-7.0.97/examples/s3/getobjectlockconfig.go000066400000000000000000000036551510244170000220360ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "log" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // Get object lock configuration. enabled, mode, validity, unit, err := s3Client.GetObjectLockConfig(context.Background(), "tbucket13a") if err != nil { log.Fatalln(err) } fmt.Printf("object lock is %v for bucket 'my-bucketname'\n", enabled) if mode != nil { fmt.Printf("%v mode is enabled for %v %v for bucket 'my-bucketname'\n", *mode, *validity, *unit) } else { fmt.Println("No mode is enabled for bucket 'my-bucketname'") } } minio-go-7.0.97/examples/s3/getobjectretention.go000066400000000000000000000033121510244170000217150ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and // my-testfile are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } m, t, err := s3Client.GetObjectRetention(context.Background(), "my-bucket", "my-object", "") if err != nil { log.Fatalln(err) } log.Println("Get object retention successful, Mode: ", m.String(), " Retainuntil Date ", t.String()) } minio-go-7.0.97/examples/s3/getobjecttagging.go000066400000000000000000000032751510244170000213360ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } tags, err := s3Client.GetObjectTagging(context.Background(), "my-bucketname", "my-objectname", minio.GetObjectTaggingOptions{}) if err != nil { log.Fatalln(err) } fmt.Printf("Fetched Object Tags: %s", tags) } minio-go-7.0.97/examples/s3/go.mod000066400000000000000000000023451510244170000166030ustar00rootroot00000000000000module github.com/minio/minio-go/examples/s3 go 1.23.0 toolchain go1.24.1 // Overridden by `replace` below, to point all versions at the local minio-go source, so version shouldn't matter here. require github.com/minio/minio-go/v7 v7.0.73 require ( github.com/cheggaaa/pb v1.0.29 github.com/minio/sio v0.3.0 golang.org/x/crypto v0.39.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.11 // indirect github.com/klauspost/crc32 v1.3.0 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/minio/crc64nvme v1.1.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rs/xid v1.6.0 // indirect github.com/tinylib/msgp v1.3.0 // indirect golang.org/x/net v0.41.0 // indirect golang.org/x/sys v0.34.0 // indirect golang.org/x/text v0.26.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/minio/minio-go/v7 => ../.. minio-go-7.0.97/examples/s3/go.sum000066400000000000000000000127751510244170000166400ustar00rootroot00000000000000github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q= github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= minio-go-7.0.97/examples/s3/healthcheck.go000066400000000000000000000032651510244170000202710ustar00rootroot00000000000000//go:build ignore // +build ignore /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "log" "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // start healthcheck on the endpoint cancelFn, err := s3Client.HealthCheck(5 * time.Second) if err == nil { defer cancelFn() } for { if s3Client.IsOnline() { // do some S3 operations } time.Sleep(1 * time.Second) } } minio-go-7.0.97/examples/s3/list-directory-buckets.go000066400000000000000000000032071510244170000224350ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2025 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID and YOUR-SECRETACCESSKEY are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3express-control.us-east-1.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } dirBuckets, err := s3Client.ListDirectoryBuckets(context.Background()) if err != nil { log.Fatalln(err) } for bucket, err := range dirBuckets { log.Println(bucket, err) } } minio-go-7.0.97/examples/s3/listbuckets.go000066400000000000000000000031301510244170000203510ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID and YOUR-SECRETACCESSKEY are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } buckets, err := s3Client.ListBuckets(context.Background()) if err != nil { log.Fatalln(err) } for _, bucket := range buckets { log.Println(bucket) } } minio-go-7.0.97/examples/s3/listincompleteuploads.go000066400000000000000000000034441510244170000224500ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // List all multipart uploads from a bucket-name with a matching prefix. for multipartObject := range s3Client.ListIncompleteUploads(context.Background(), "my-bucketname", "my-prefixname", true) { if multipartObject.Err != nil { fmt.Println(multipartObject.Err) return } fmt.Println(multipartObject) } return } minio-go-7.0.97/examples/s3/listobjects-N.go000066400000000000000000000047001510244170000205410ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { fmt.Println(err) return } // List 'N' number of objects from a bucket-name with a matching prefix. listObjectsN := func(bucket, prefix string, recursive bool, N int) (objsInfo []minio.ObjectInfo, err error) { ctx, cancel := context.WithCancel(context.Background()) // Indicate ListObjects go-routine to exit and stop feeding the objectInfo channel. defer cancel() i := 1 opts := minio.ListObjectsOptions{ UseV1: true, Prefix: prefix, Recursive: recursive, } for object := range s3Client.ListObjects(ctx, bucket, opts) { if object.Err != nil { return nil, object.Err } i++ // Verify if we have printed N objects. if i == N { break } objsInfo = append(objsInfo, object) } return objsInfo, nil } // List recursively first 100 entries for prefix 'my-prefixname'. recursive := true objsInfo, err := listObjectsN("my-bucketname", "my-prefixname", recursive, 100) if err != nil { fmt.Println(err) } // Print all the entries. fmt.Println(objsInfo) } minio-go-7.0.97/examples/s3/listobjects.go000066400000000000000000000035101510244170000203440ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { fmt.Println(err) return } opts := minio.ListObjectsOptions{ UseV1: true, Prefix: "my-prefixname", Recursive: true, } // List all objects from a bucket-name with a matching prefix. for object := range s3Client.ListObjects(context.Background(), "my-bucketname", opts) { if object.Err != nil { fmt.Println(object.Err) return } fmt.Println(object) } return } minio-go-7.0.97/examples/s3/listobjectsV2.go000066400000000000000000000034651510244170000205650ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { fmt.Println(err) return } opts := minio.ListObjectsOptions{ Recursive: true, Prefix: "my-prefixname", } // List all objects from a bucket-name with a matching prefix. for object := range s3Client.ListObjects(context.Background(), "my-bucketname", opts) { if object.Err != nil { fmt.Println(object.Err) return } fmt.Println(object) } return } minio-go-7.0.97/examples/s3/listobjectversions.go000066400000000000000000000035501510244170000217560ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { fmt.Println(err) return } opts := minio.ListObjectsOptions{ WithVersions: true, Prefix: "my-prefixname", Recursive: true, } // List all objects from a bucket-name with a matching prefix. for objectVersion := range s3Client.ListObjects(context.Background(), "my-bucketname", opts) { if objectVersion.Err != nil { fmt.Println(objectVersion.Err) return } fmt.Println(objectVersion) } return } minio-go-7.0.97/examples/s3/makebucket.go000066400000000000000000000032151510244170000201340ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } opts := minio.MakeBucketOptions{ Region: "us-east-1", } err = s3Client.MakeBucket(context.Background(), "my-bucketname", opts) if err != nil { log.Fatalln(err) } log.Println("Success") } minio-go-7.0.97/examples/s3/presignedgetobject.go000066400000000000000000000036031510244170000216710ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "net/url" "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // Set request parameters reqParams := make(url.Values) reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") // Gernerate presigned get object url. presignedURL, err := s3Client.PresignedGetObject(context.Background(), "my-bucketname", "my-objectname", time.Duration(1000)*time.Second, reqParams) if err != nil { log.Fatalln(err) } log.Println(presignedURL) } minio-go-7.0.97/examples/s3/presignedheadobject.go000066400000000000000000000036041510244170000220140ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "net/url" "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // Set request parameters reqParams := make(url.Values) reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") // Gernerate presigned get object url. presignedURL, err := s3Client.PresignedHeadObject(context.Background(), "my-bucketname", "my-objectname", time.Duration(1000)*time.Second, reqParams) if err != nil { log.Fatalln(err) } log.Println(presignedURL) } minio-go-7.0.97/examples/s3/presignedpostpolicy.go000066400000000000000000000037311510244170000221320ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "log" "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } policy := minio.NewPostPolicy() policy.SetBucket("my-bucketname") policy.SetKey("my-objectname") // Expires in 10 days. policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // Returns form data for POST form request. url, formData, err := s3Client.PresignedPostPolicy(context.Background(), policy) if err != nil { log.Fatalln(err) } fmt.Printf("curl ") for k, v := range formData { fmt.Printf("-F %s=%s ", k, v) } fmt.Printf("-F file=@/etc/bash.bashrc ") fmt.Printf("%s\n", url) } minio-go-7.0.97/examples/s3/presignedputobject.go000066400000000000000000000032551510244170000217250ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } presignedURL, err := s3Client.PresignedPutObject(context.Background(), "my-bucketname", "my-objectname", time.Duration(1000)*time.Second) if err != nil { log.Fatalln(err) } log.Println(presignedURL) } minio-go-7.0.97/examples/s3/put-encrypted-object.go000066400000000000000000000051241510244170000220710ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "os" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } filePath := "my-testfile" // Specify a local file that we will upload // Open a local file that we will upload file, err := os.Open(filePath) if err != nil { log.Fatalln(err) } defer file.Close() // Get file stats. fstat, err := file.Stat() if err != nil { log.Fatalln(err) } bucketname := "my-bucketname" // Specify a bucket name - the bucket must already exist objectName := "my-objectname" // Specify a object name password := "correct horse battery staple" // Specify your password. DO NOT USE THIS ONE - USE YOUR OWN. // New SSE-C where the cryptographic key is derived from a password and the objectname + bucketname as salt encryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketname+objectName)) // Encrypt file content and upload to the server n, err := s3Client.PutObject(context.Background(), bucketname, objectName, file, fstat.Size(), minio.PutObjectOptions{ServerSideEncryption: encryption}) if err != nil { log.Fatalln(err) } log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") } minio-go-7.0.97/examples/s3/putbucketcors.go000066400000000000000000000044041510244170000207170ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2024 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/cors" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-prefixname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } bucket := "my-bucket-name" corsRules := []cors.Rule{ { AllowedHeader: []string{"*"}, AllowedMethod: []string{"GET", "PUT"}, AllowedOrigin: []string{"https://example.com"}, }, } corsConfig := cors.NewConfig(corsRules) err = s3Client.SetBucketCors(context.Background(), bucket, corsConfig) if err != nil { log.Fatalln(fmt.Errorf("Error setting bucket cors: %v", err)) } retCors, err := s3Client.GetBucketCors(context.Background(), bucket) if err != nil { log.Fatalln(fmt.Errorf("Error getting bucket cors: %v", err)) } fmt.Printf("Returned Bucket CORS configuration: %+v\n", retCors) err = s3Client.SetBucketCors(context.Background(), bucket, nil) if err != nil { log.Fatalln(fmt.Errorf("Error removing bucket cors: %v", err)) } } minio-go-7.0.97/examples/s3/putobject-checksum.go000066400000000000000000000047541510244170000216310ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2023 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "encoding/base64" "hash/crc32" "io" "log" "os" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } object, err := os.Open("putobject-checksum.go") if err != nil { log.Fatalln(err) } defer object.Close() objectStat, err := object.Stat() if err != nil { log.Fatalln(err) } // Create CRC32C: crc := crc32.New(crc32.MakeTable(crc32.Castagnoli)) _, err = io.Copy(crc, object) if err != nil { log.Fatalln(err) } meta := map[string]string{"x-amz-checksum-crc32c": base64.StdEncoding.EncodeToString(crc.Sum(nil))} // Reset object. _, err = object.Seek(0, io.SeekStart) if err != nil { log.Fatalln(err) } // Upload object. // Checksums are different with multipart, so we disable that. info, err := s3Client.PutObject(context.Background(), "my-bucket", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream", UserMetadata: meta, DisableMultipart: true}) if err != nil { log.Fatalln(err) } log.Println("Uploaded", "my-objectname", "of size:", info.Size, "with CRC32C:", info.ChecksumCRC32C, "successfully.") } minio-go-7.0.97/examples/s3/putobject-client-encryption.go000066400000000000000000000046221510244170000234670ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "os" "path" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/sio" "golang.org/x/crypto/argon2" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } object, err := os.Open("my-testfile") if err != nil { log.Fatalln(err) } defer object.Close() objectStat, err := object.Stat() if err != nil { log.Fatalln(err) } password := []byte("myfavoritepassword") // Change as per your needs. salt := []byte(path.Join("my-bucketname", "my-objectname")) // Change as per your needs. encrypted, err := sio.EncryptReader(object, sio.Config{ // generate a 256 bit long key. Key: argon2.IDKey(password, salt, 1, 64*1024, 4, 32), }) if err != nil { log.Fatalln(err) } encSize, err := sio.EncryptedSize(uint64(objectStat.Size())) if err != nil { log.Fatalln(err) } _, err = s3Client.PutObject(context.Background(), "my-bucketname", "my-objectname", encrypted, int64(encSize), minio.PutObjectOptions{}) if err != nil { log.Fatalln(err) } log.Println("Successfully encrypted 'my-objectname'") } minio-go-7.0.97/examples/s3/putobject-getobject-sse.go000066400000000000000000000042551510244170000225610ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "bytes" "context" "io" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. minioClient, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } bucketName := "my-bucket" objectName := "my-encrypted-object" object := []byte("Hello again") encryption := encrypt.DefaultPBKDF([]byte("my secret password"), []byte(bucketName+objectName)) _, err = minioClient.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(object), int64(len(object)), minio.PutObjectOptions{ ServerSideEncryption: encryption, }) if err != nil { log.Fatalln(err) } reader, err := minioClient.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: encryption}) if err != nil { log.Fatalln(err) } defer reader.Close() decBytes, err := io.ReadAll(reader) if err != nil { log.Fatalln(err) } if !bytes.Equal(decBytes, object) { log.Fatalln("Expected %s, got %s", string(object), string(decBytes)) } } minio-go-7.0.97/examples/s3/putobject-progress.go000066400000000000000000000043141510244170000216630ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/cheggaaa/pb" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } reader, err := s3Client.GetObject(context.Background(), "my-bucketname", "my-objectname", minio.GetObjectOptions{}) if err != nil { log.Fatalln(err) } defer reader.Close() objectInfo, err := reader.Stat() if err != nil { log.Fatalln(err) } // Progress reader is notified as PutObject makes progress with // the Reads inside. progress := pb.New64(objectInfo.Size) progress.Start() n, err := s3Client.PutObject(context.Background(), "my-bucketname", "my-objectname-progress", reader, objectInfo.Size, minio.PutObjectOptions{ContentType: "application/octet-stream", Progress: progress}) if err != nil { log.Fatalln(err) } log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") } minio-go-7.0.97/examples/s3/putobject-s3-accelerate.go000066400000000000000000000040551510244170000224340ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "os" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // Enable S3 transfer accelerate endpoint. s3Client.SetS3TransferAccelerate("s3-accelerate.amazonaws.com") object, err := os.Open("my-testfile") if err != nil { log.Fatalln(err) } defer object.Close() objectStat, err := object.Stat() if err != nil { log.Fatalln(err) } n, err := s3Client.PutObject(context.Background(), "my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { log.Fatalln(err) } log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") } minio-go-7.0.97/examples/s3/putobject-streaming.go000066400000000000000000000035061510244170000220120ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "os" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } object, err := os.Open("my-testfile") if err != nil { log.Fatalln(err) } defer object.Close() n, err := s3Client.PutObject(context.Background(), "my-bucketname", "my-objectname", object, -1, minio.PutObjectOptions{}) if err != nil { log.Fatalln(err) } log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") } minio-go-7.0.97/examples/s3/putobject-with-tags.go000066400000000000000000000040171510244170000217260ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "os" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } object, err := os.Open("my-testfile") if err != nil { log.Fatalln(err) } defer object.Close() objectStat, err := object.Stat() if err != nil { log.Fatalln(err) } tags := map[string]string{ "Tag1": "Value1", "Tag2": "Value2", } n, err := s3Client.PutObject(context.Background(), "my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream", UserTags: tags}) if err != nil { log.Fatalln(err) } log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") } minio-go-7.0.97/examples/s3/putobject.go000066400000000000000000000037111510244170000200210ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "os" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } object, err := os.Open("my-testfile") if err != nil { log.Fatalln(err) } defer object.Close() objectStat, err := object.Stat() if err != nil { log.Fatalln(err) } info, err := s3Client.PutObject(context.Background(), "my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { log.Fatalln(err) } log.Println("Uploaded", "my-objectname", " of size: ", info.Size, "Successfully.") } minio-go-7.0.97/examples/s3/putobjectlegalhold.go000066400000000000000000000033701510244170000216760ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and // my-testfile are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } s := minio.LegalHoldEnabled opts := minio.PutObjectLegalHoldOptions{ Status: &s, } err = s3Client.PutObjectLegalHold(context.Background(), "my-bucket", "my-object", opts) if err != nil { log.Fatalln(err) } log.Println("Set object legal-hold on my-object successfully.") } minio-go-7.0.97/examples/s3/putobjectretention.go000066400000000000000000000036121510244170000217510ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and // my-testfile are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } t := time.Date(2020, time.November, 18, 14, 0, 0, 0, time.UTC) m := minio.RetentionMode(minio.Governance) opts := minio.PutObjectRetentionOptions{ GovernanceBypass: true, RetainUntilDate: &t, Mode: &m, } err = s3Client.PutObjectRetention(context.Background(), "my-bucket", "my-object", opts) if err != nil { log.Fatalln(err) } log.Println("Set object retention on my-object successfully.") } minio-go-7.0.97/examples/s3/putobjecttagging.go000066400000000000000000000035041510244170000213620ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/tags" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } tagMap := map[string]string{ "Tag1": "Value1", "Tag2": "Value2", } t, err := tags.MapToObjectTags(tagMap) if err != nil { log.Fatalln(err) } err = s3Client.PutObjectTagging(context.Background(), "my-bucketname", "my-objectname", t, minio.PutObjectTaggingOptions{}) if err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/removeallbucketnotification.go000066400000000000000000000032431510244170000236150ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) err = s3Client.RemoveAllBucketNotification(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } log.Println("Bucket notification are successfully removed.") } minio-go-7.0.97/examples/s3/removebucket.go000066400000000000000000000032101510244170000205070ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // This operation will only work if your bucket is empty. err = s3Client.RemoveBucket(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } log.Println("Success") } minio-go-7.0.97/examples/s3/removebucketencryption.go000066400000000000000000000032261510244170000226310ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // Get default encryption configuration set on a S3 bucket err = s3Client.RemoveBucketEncryption(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/removebucketreplication.go000066400000000000000000000032141510244170000227450ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // Remove replication configuration on a bucket err = s3Client.RemoveBucketReplication(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/removebuckettagging.go000066400000000000000000000031271510244170000220570ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } err = s3Client.RemoveBucketTagging(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/removeincompleteupload.go000066400000000000000000000031661510244170000226100ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } err = s3Client.RemoveIncompleteUpload(context.Background(), "my-bucketname", "my-objectname") if err != nil { log.Fatalln(err) } log.Println("Success") } minio-go-7.0.97/examples/s3/removeobject.go000066400000000000000000000032651510244170000205120ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } opts := minio.RemoveObjectOptions{ GovernanceBypass: true, } err = s3Client.RemoveObject(context.Background(), "my-bucketname", "my-objectname", opts) if err != nil { log.Fatalln(err) } log.Println("Success") } minio-go-7.0.97/examples/s3/removeobjects.go000066400000000000000000000043641510244170000206760ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } objectsCh := make(chan minio.ObjectInfo) // Send object names that are needed to be removed to objectsCh go func() { defer close(objectsCh) // List all objects from a bucket-name with a matching prefix. opts := minio.ListObjectsOptions{Prefix: "my-prefixname", Recursive: true} for object := range s3Client.ListObjects(context.Background(), "my-bucketname", opts) { if object.Err != nil { log.Fatalln(object.Err) } objectsCh <- object } }() // Call RemoveObjects API errorCh := s3Client.RemoveObjects(context.Background(), "my-bucketname", objectsCh, minio.RemoveObjectsOptions{}) // Print errors received from RemoveObjects API for e := range errorCh { log.Fatalln("Failed to remove " + e.ObjectName + ", error: " + e.Err.Error()) } log.Println("Success") } minio-go-7.0.97/examples/s3/removeobjecttagging.go000066400000000000000000000032061510244170000220460ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } err = s3Client.RemoveObjectTagging(context.Background(), "my-bucketname", "my-objectname", minio.RemoveObjectTaggingOptions{}) if err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/restoreobject-select.go000066400000000000000000000045251510244170000221550ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "log" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and // my-testfile are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } opts := minio.RestoreRequest{} opts.SetType(minio.RestoreSelect) opts.SetTier(minio.TierStandard) selectParameters := minio.SelectParameters{ Expression: "SELECT * FROM object", ExpressionType: minio.QueryExpressionTypeSQL, InputSerialization: minio.SelectObjectInputSerialization{ CSV: &minio.CSVInputOptions{ FileHeaderInfo: minio.CSVFileHeaderInfoUse, }, }, OutputSerialization: minio.SelectObjectOutputSerialization{ CSV: &minio.CSVOutputOptions{}, }, } opts.SetSelectParameters(selectParameters) outputLocation := minio.OutputLocation{S3: minio.S3{BucketName: "your-bucket", Prefix: "sql-request-output.csv"}} opts.SetOutputLocation(outputLocation) err = s3Client.RestoreObject(context.Background(), "your-bucket", "input.csv", "", opts) if err != nil { log.Fatalln(err) } fmt.Println("Restore SQL request Succeeded.") } minio-go-7.0.97/examples/s3/restoreobject.go000066400000000000000000000034421510244170000206750ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "fmt" "log" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and // my-testfile are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } opts := minio.RestoreRequest{} opts.SetDays(1) opts.SetGlacierJobParameters(minio.GlacierJobParameters{Tier: minio.TierStandard}) err = s3Client.RestoreObject(context.Background(), "your-bucket", "your-object", "", opts) if err != nil { log.Fatalln(err) } fmt.Println("Restore request succeeded.") } minio-go-7.0.97/examples/s3/selectobject.go000066400000000000000000000044131510244170000204700ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "io" "log" "os" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and // my-testfile are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } opts := minio.SelectObjectOptions{ Expression: "select count(*) from s3object", ExpressionType: minio.QueryExpressionTypeSQL, InputSerialization: minio.SelectObjectInputSerialization{ CompressionType: minio.SelectCompressionNONE, CSV: &minio.CSVInputOptions{ FileHeaderInfo: minio.CSVFileHeaderInfoNone, RecordDelimiter: "\n", FieldDelimiter: ",", }, }, OutputSerialization: minio.SelectObjectOutputSerialization{ CSV: &minio.CSVOutputOptions{ RecordDelimiter: "\n", FieldDelimiter: ",", }, }, } reader, err := s3Client.SelectObjectContent(context.Background(), "mycsvbucket", "mycsv.csv", opts) if err != nil { log.Fatalln(err) } defer reader.Close() if _, err := io.Copy(os.Stdout, reader); err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/setbucketencryption.go000066400000000000000000000033211510244170000221230ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/sse" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // Set default encryption configuration on a bucket err = s3Client.SetBucketEncryption(context.Background(), "my-bucketname", sse.NewConfigurationSSES3()) if err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/setbucketlifecycle.go000066400000000000000000000035611510244170000216760ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/lifecycle" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // Set lifecycle on a bucket config := lifecycle.NewConfiguration() config.Rules = []lifecycle.Rule{ { ID: "expire-bucket", Status: "Enabled", Expiration: lifecycle.Expiration{ Days: 365, }, }, } err = s3Client.SetBucketLifecycle(context.Background(), "my-bucketname", config) if err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/setbucketnotification.go000066400000000000000000000066271510244170000224330ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/notification" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // ARN represents a notification channel that needs to be created in your S3 provider // (e.g. http://docs.aws.amazon.com/sns/latest/dg/CreateTopic.html) // An example of an ARN: // arn:aws:sns:us-east-1:804064459714:UploadPhoto // ^ ^ ^ ^ ^ // Provider __| | | | | // | Region Account ID |_ Notification Name // Service _| // // You should replace YOUR-PROVIDER, YOUR-SERVICE, YOUR-REGION, YOUR-ACCOUNT-ID and YOUR-RESOURCE // with actual values that you receive from the S3 provider // Here you create a new Topic notification topicArn := notification.NewArn("YOUR-PROVIDER", "YOUR-SERVICE", "YOUR-REGION", "YOUR-ACCOUNT-ID", "YOUR-RESOURCE") topicConfig := notification.NewConfig(topicArn) topicConfig.AddEvents(notification.ObjectCreatedAll, notification.ObjectRemovedAll) topicConfig.AddFilterPrefix("photos/") topicConfig.AddFilterSuffix(".jpg") // Create a new Queue notification queueArn := notification.NewArn("YOUR-PROVIDER", "YOUR-SERVICE", "YOUR-REGION", "YOUR-ACCOUNT-ID", "YOUR-RESOURCE") queueConfig := notification.NewConfig(queueArn) queueConfig.AddEvents(notification.ObjectRemovedAll) // Create a new Lambda (CloudFunction) lambdaArn := notification.NewArn("YOUR-PROVIDER", "YOUR-SERVICE", "YOUR-REGION", "YOUR-ACCOUNT-ID", "YOUR-RESOURCE") lambdaConfig := notification.NewConfig(lambdaArn) lambdaConfig.AddEvents(notification.ObjectRemovedAll) lambdaConfig.AddFilterSuffix(".swp") // Now, set all previously created notification configs config := notification.Configuration{} config.AddTopic(topicConfig) config.AddQueue(queueConfig) config.AddLambda(lambdaConfig) err = s3Client.SetBucketNotification(context.Background(), "YOUR-BUCKET", config) if err != nil { log.Fatalln("Error: " + err.Error()) } log.Println("Success") } minio-go-7.0.97/examples/s3/setbucketpolicy.go000066400000000000000000000034531510244170000212360ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // Create policy policy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::my-bucketname/*"],"Sid": ""}]}` err = s3Client.SetBucketPolicy(context.Background(), "my-bucketname", policy) if err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/setbucketreplication.go000066400000000000000000000047111510244170000222460ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "encoding/xml" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/replication" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) replicationStr := `stringEnabled1Disabledarn:aws:s3:::destPrefixTag-Key1Tag-Value1Tag-Key2Tag-Value2` var replCfg replication.Config err = xml.Unmarshal([]byte(replicationStr), &replCfg) if err != nil { log.Fatalln(err) } // This replication ARN should have been generated for replication endpoint using `mc admin bucket remote` command replCfg.Role = "arn:minio:replica::dadddae7-f1d7-440f-b5d6-651aa9a8c8a7:dest" // Set replication config on a bucket err = s3Client.SetBucketReplication(context.Background(), "my-bucketname", replCfg) if err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/setbuckettagging.go000066400000000000000000000034101510244170000213500ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/tags" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and // my-objectname are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } t, err := tags.MapToBucketTags(map[string]string{ "Tag1": "Value1", "Tag2": "Value2", }) if err != nil { log.Fatalln(err) } err = s3Client.SetBucketTagging(context.Background(), "my-bucketname", t) if err != nil { log.Fatalln(err) } } minio-go-7.0.97/examples/s3/setobjectlockconfig.go000066400000000000000000000033661510244170000220510ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } // s3Client.TraceOn(os.Stderr) // Set object lock configuration. mode := minio.Governance validity := uint(30) unit := minio.Days err = s3Client.SetObjectLockConfig(context.Background(), "my-bucketname", &mode, &validity, &unit) if err != nil { log.Fatalln(err) } log.Println("Success") } minio-go-7.0.97/examples/s3/statobject.go000066400000000000000000000032061510244170000201630ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname // are dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } stat, err := s3Client.StatObject(context.Background(), "my-bucketname", "my-objectname", minio.StatObjectOptions{}) if err != nil { log.Fatalln(err) } log.Println(stat) } minio-go-7.0.97/examples/s3/suspendversioning.go000066400000000000000000000031231510244170000216040ustar00rootroot00000000000000//go:build example // +build example /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2019 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "context" "log" minio "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are // dummy values, please replace them with original values. // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. // This boolean value is the last argument for New(). // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. s3Client, err := minio.New("s3.amazonaws.com", &minio.Options{ Creds: credentials.NewStaticV4("YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", ""), Secure: true, }) if err != nil { log.Fatalln(err) } err = s3Client.SuspendVersioning(context.Background(), "my-bucketname") if err != nil { log.Fatalln(err) } log.Println("Disabled") } minio-go-7.0.97/functional_tests.go000066400000000000000000016666231510244170000172650ustar00rootroot00000000000000//go:build mint // +build mint /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "archive/zip" "bytes" "context" "crypto/sha256" "encoding/base64" "errors" "fmt" "hash" "hash/crc32" "io" "iter" "log/slog" "math/rand" "mime/multipart" "net/http" "net/url" "os" "path" "path/filepath" "reflect" "runtime" "sort" "strconv" "strings" "sync" "time" "github.com/dustin/go-humanize" "github.com/google/uuid" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/cors" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/notification" "github.com/minio/minio-go/v7/pkg/tags" ) const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" const ( letterIdxBits = 6 // 6 bits to represent a letter index letterIdxMask = 1<= len(buf) { err = nil } else if n > 0 && err == io.EOF { err = io.ErrUnexpectedEOF } return n, err } func baseLogger(testName, function string, args map[string]interface{}, startTime time.Time) *slog.Logger { // calculate the test case duration duration := time.Since(startTime) // log with the fields as per mint l := slog.With( "name", "minio-go: "+testName, "duration", duration.Nanoseconds()/1000000, ) if function != "" { l = l.With("function", function) } if len(args) > 0 { l = l.With("args", args) } return l } // log successful test runs func logSuccess(testName, function string, args map[string]interface{}, startTime time.Time) { baseLogger(testName, function, args, startTime). With("status", "PASS"). Info("") } // As few of the features are not available in Gateway(s) currently, Check if err value is NotImplemented, // and log as NA in that case and continue execution. Otherwise log as failure and return func logError(testName, function string, args map[string]interface{}, startTime time.Time, alert, message string, err error) { // If server returns NotImplemented we assume it is gateway mode and hence log it as info and move on to next tests // Special case for ComposeObject API as it is implemented on client side and adds specific error details like `Error in upload-part-copy` in // addition to NotImplemented error returned from server if isErrNotImplemented(err) { logIgnored(testName, function, args, startTime, message) } else { logFailure(testName, function, args, startTime, alert, message, err) if !isRunOnFail() { panic(fmt.Sprintf("Test failed with message: %s, err: %v", message, err)) } } } // Log failed test runs, do not call this directly, use logError instead, as that correctly stops the test run func logFailure(testName, function string, args map[string]interface{}, startTime time.Time, alert, message string, err error) { l := baseLogger(testName, function, args, startTime).With( "status", "FAIL", "alert", alert, "message", message, ) if err != nil { l = l.With("error", err) } l.Error("") } // log not applicable test runs func logIgnored(testName, function string, args map[string]interface{}, startTime time.Time, alert string) { baseLogger(testName, function, args, startTime). With( "status", "NA", "alert", strings.Split(alert, " ")[0]+" is NotImplemented", ).Info("") } // Delete objects in given bucket, recursively func cleanupBucket(bucketName string, c *minio.Client) error { // Create a done channel to control 'ListObjectsV2' go routine. doneCh := make(chan struct{}) // Exit cleanly upon return. defer close(doneCh) // Iterate over all objects in the bucket via listObjectsV2 and delete for objCh := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{Recursive: true}) { if objCh.Err != nil { return objCh.Err } if objCh.Key != "" { err := c.RemoveObject(context.Background(), bucketName, objCh.Key, minio.RemoveObjectOptions{}) if err != nil { return err } } } for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true) { if objPartInfo.Err != nil { return objPartInfo.Err } if objPartInfo.Key != "" { err := c.RemoveIncompleteUpload(context.Background(), bucketName, objPartInfo.Key) if err != nil { return err } } } // objects are already deleted, clear the buckets now return c.RemoveBucket(context.Background(), bucketName) } func cleanupVersionedBucket(bucketName string, c *minio.Client) error { doneCh := make(chan struct{}) defer close(doneCh) for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) { if obj.Err != nil { return obj.Err } if obj.Key != "" { err := c.RemoveObject(context.Background(), bucketName, obj.Key, minio.RemoveObjectOptions{VersionID: obj.VersionID, GovernanceBypass: true}) if err != nil { return err } } } for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true) { if objPartInfo.Err != nil { return objPartInfo.Err } if objPartInfo.Key != "" { err := c.RemoveIncompleteUpload(context.Background(), bucketName, objPartInfo.Key) if err != nil { return err } } } // objects are already deleted, clear the buckets now err := c.RemoveBucket(context.Background(), bucketName) if err != nil { for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) { slog.Info("found object", "key", obj.Key, "version", obj.VersionID) } } return err } func isErrNotImplemented(err error) bool { return minio.ToErrorResponse(err).Code == minio.NotImplemented } func isRunOnFail() bool { return os.Getenv("RUN_ON_FAIL") == "1" } func init() { // If server endpoint is not set, all tests default to // using https://play.min.io if os.Getenv(serverEndpoint) == "" { os.Setenv(serverEndpoint, "play.min.io") os.Setenv(accessKey, "Q3AM3UQ867SPQQA43P2F") os.Setenv(secretKey, "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG") os.Setenv(enableHTTPS, "1") } } var mintDataDir = os.Getenv("MINT_DATA_DIR") func getMintDataDirFilePath(filename string) (fp string) { if mintDataDir == "" { return fp } return filepath.Join(mintDataDir, filename) } func newRandomReader(seed, size int64) io.Reader { return io.LimitReader(rand.New(rand.NewSource(seed)), size) } func mustCrcReader(r io.Reader) uint32 { crc := crc32.NewIEEE() _, err := io.Copy(crc, r) if err != nil { panic(err) } return crc.Sum32() } func crcMatches(r io.Reader, want uint32) error { crc := crc32.NewIEEE() _, err := io.Copy(crc, r) if err != nil { panic(err) } got := crc.Sum32() if got != want { return fmt.Errorf("crc mismatch, want %x, got %x", want, got) } return nil } func crcMatchesName(r io.Reader, name string) error { want := dataFileCRC32[name] crc := crc32.NewIEEE() _, err := io.Copy(crc, r) if err != nil { panic(err) } got := crc.Sum32() if got != want { return fmt.Errorf("crc mismatch, want %x, got %x", want, got) } return nil } // read data from file if it exists or optionally create a buffer of particular size func getDataReader(fileName string) io.ReadCloser { if mintDataDir == "" { size := int64(dataFileMap[fileName]) if _, ok := dataFileCRC32[fileName]; !ok { dataFileCRC32[fileName] = mustCrcReader(newRandomReader(size, size)) } return io.NopCloser(newRandomReader(size, size)) } reader, _ := os.Open(getMintDataDirFilePath(fileName)) if _, ok := dataFileCRC32[fileName]; !ok { dataFileCRC32[fileName] = mustCrcReader(reader) reader.Close() reader, _ = os.Open(getMintDataDirFilePath(fileName)) } return reader } // randString generates random names and prepends them with a known prefix. func randString(n int, src rand.Source, prefix string) string { b := make([]byte, n) // A rand.Int63() generates 63 random bits, enough for letterIdxMax letters! for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; { if remain == 0 { cache, remain = src.Int63(), letterIdxMax } if idx := int(cache & letterIdxMask); idx < len(letterBytes) { b[i] = letterBytes[idx] i-- } cache >>= letterIdxBits remain-- } return prefix + string(b[0:30-len(prefix)]) } var dataFileMap = map[string]int{ "datafile-0-b": 0, "datafile-1-b": 1, "datafile-1-kB": 1 * humanize.KiByte, "datafile-10-kB": 10 * humanize.KiByte, "datafile-33-kB": 33 * humanize.KiByte, "datafile-100-kB": 100 * humanize.KiByte, "datafile-1.03-MB": 1056 * humanize.KiByte, "datafile-1-MB": 1 * humanize.MiByte, "datafile-5-MB": 5 * humanize.MiByte, "datafile-6-MB": 6 * humanize.MiByte, "datafile-11-MB": 11 * humanize.MiByte, "datafile-65-MB": 65 * humanize.MiByte, "datafile-129-MB": 129 * humanize.MiByte, } var dataFileCRC32 = map[string]uint32{} func isFullMode() bool { return os.Getenv("MINT_MODE") == "full" } func getFuncName() string { return getFuncNameLoc(2) } func getFuncNameLoc(caller int) string { pc, _, _, _ := runtime.Caller(caller) return strings.TrimPrefix(runtime.FuncForPC(pc).Name(), "main.") } type ClientConfig struct { // MinIO client configuration TraceOn bool // Turn on tracing of HTTP requests and responses to stderr CredsV2 bool // Use V2 credentials if true, otherwise use v4 TrailingHeaders bool // Send trailing headers in requests } func NewClient(config ClientConfig) (*minio.Client, error) { // Instantiate new MinIO client var creds *credentials.Credentials if config.CredsV2 { creds = credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), "") } else { creds = credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), "") } opts := &minio.Options{ Creds: creds, Transport: createHTTPTransport(), Secure: mustParseBool(os.Getenv(enableHTTPS)), TrailingHeaders: config.TrailingHeaders, } client, err := minio.New(os.Getenv(serverEndpoint), opts) if err != nil { return nil, err } if config.TraceOn { client.TraceOn(os.Stderr) } // Set user agent. client.SetAppInfo("MinIO-go-FunctionalTest", appVersion) return client, nil } // Tests bucket re-create errors. func testMakeBucketError() { region := "eu-central-1" // initialize logging params startTime := time.Now() testName := getFuncName() function := "MakeBucket(bucketName, region)" // initialize logging params args := map[string]interface{}{ "bucketName": "", "region": region, } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket in 'eu-central-1'. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket Failed", err) return } defer cleanupBucket(bucketName, c) if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err == nil { logError(testName, function, args, startTime, "", "Bucket already exists", err) return } // Verify valid error response from server. if minio.ToErrorResponse(err).Code != minio.BucketAlreadyExists && minio.ToErrorResponse(err).Code != minio.BucketAlreadyOwnedByYou { logError(testName, function, args, startTime, "", "Invalid error returned by server", err) return } logSuccess(testName, function, args, startTime) } func testMetadataSizeLimit() { startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, objectSize, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts.UserMetadata": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return } bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) const HeaderSizeLimit = 8 * 1024 const UserMetadataLimit = 2 * 1024 // Meta-data greater than the 2 KB limit of AWS - PUT calls with this meta-data should fail metadata := make(map[string]string) metadata["X-Amz-Meta-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+UserMetadataLimit-len("X-Amz-Meta-Mint-Test"))) args["metadata"] = fmt.Sprint(metadata) _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) if err == nil { logError(testName, function, args, startTime, "", "Created object with user-defined metadata exceeding metadata size limits", nil) return } // Meta-data (headers) greater than the 8 KB limit of AWS - PUT calls with this meta-data should fail metadata = make(map[string]string) metadata["X-Amz-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+HeaderSizeLimit-len("X-Amz-Mint-Test"))) args["metadata"] = fmt.Sprint(metadata) _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) if err == nil { logError(testName, function, args, startTime, "", "Created object with headers exceeding header size limits", nil) return } logSuccess(testName, function, args, startTime) } // Tests various bucket supported formats. func testMakeBucketRegions() { region := "eu-central-1" // initialize logging params startTime := time.Now() testName := getFuncName() function := "MakeBucket(bucketName, region)" // initialize logging params args := map[string]interface{}{ "bucketName": "", "region": region, } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket in 'eu-central-1'. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } // Make a new bucket with '.' in its name, in 'us-west-2'. This // request is internally staged into a path style instead of // virtual host style. region = "us-west-2" args["region"] = region if err = c.MakeBucket(context.Background(), bucketName+".withperiod", minio.MakeBucketOptions{Region: region}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } // Delete all objects and buckets if err = cleanupBucket(bucketName+".withperiod", c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Test PutObject using a large data to trigger multipart readat func testPutObjectReadAt() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "objectContentType", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Object content type objectContentType := "binary/octet-stream" args["objectContentType"] = objectContentType _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: objectContentType}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Get Object failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat Object failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", fmt.Sprintf("Number of bytes in stat does not match, expected %d got %d", bufSize, st.Size), err) return } if st.ContentType != objectContentType && st.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "Content types don't match", err) return } if err := crcMatchesName(r, "datafile-129-MB"); err != nil { logError(testName, function, args, startTime, "", "data CRC check failed", err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", "Object Close failed", err) return } if err := r.Close(); err == nil { logError(testName, function, args, startTime, "", "Object is already closed, didn't return error on Close", err) return } logSuccess(testName, function, args, startTime) } func testListObjectVersions() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ListObjectVersions(bucketName, prefix, recursive)" args := map[string]interface{}{ "bucketName": "", "prefix": "", "recursive": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName bufSize := dataFileMap["datafile-10-kB"] reader := getDataReader("datafile-10-kB") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } reader.Close() bufSize = dataFileMap["datafile-1-b"] reader = getDataReader("datafile-1-b") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } reader.Close() err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Unexpected object deletion", err) return } var deleteMarkers, versions int objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } if info.Key != objectName { logError(testName, function, args, startTime, "", "Unexpected object name in listing objects", nil) return } if info.VersionID == "" { logError(testName, function, args, startTime, "", "Unexpected version id in listing objects", nil) return } if info.IsDeleteMarker { deleteMarkers++ if !info.IsLatest { logError(testName, function, args, startTime, "", "Unexpected IsLatest field in listing objects", nil) return } } else { versions++ } } if deleteMarkers != 1 { logError(testName, function, args, startTime, "", "Unexpected number of DeleteMarker elements in listing objects", nil) return } if versions != 2 { logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) return } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testStatObjectWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "StatObject" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName bufSize := dataFileMap["datafile-10-kB"] reader := getDataReader("datafile-10-kB") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } reader.Close() bufSize = dataFileMap["datafile-1-b"] reader = getDataReader("datafile-1-b") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } reader.Close() objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var results []minio.ObjectInfo for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } results = append(results, info) } if len(results) != 2 { logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) return } for i := 0; i < len(results); i++ { opts := minio.StatObjectOptions{VersionID: results[i].VersionID} statInfo, err := c.StatObject(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "error during HEAD object", err) return } if statInfo.VersionID == "" || statInfo.VersionID != results[i].VersionID { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected version id", err) return } if statInfo.ETag != results[i].ETag { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err) return } if statInfo.LastModified.Unix() != results[i].LastModified.Unix() { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err) return } if statInfo.Size != results[i].Size { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err) return } } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testGetObjectWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Save the contents of datafiles to check with GetObject() reader output later var buffers [][]byte testFiles := []string{"datafile-1-b", "datafile-10-kB"} for _, testFile := range testFiles { r := getDataReader(testFile) buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return } r.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } buffers = append(buffers, buf) } objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var results []minio.ObjectInfo for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } results = append(results, info) } if len(results) != 2 { logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) return } sort.SliceStable(results, func(i, j int) bool { return results[i].Size < results[j].Size }) sort.SliceStable(buffers, func(i, j int) bool { return len(buffers[i]) < len(buffers[j]) }) for i := 0; i < len(results); i++ { opts := minio.GetObjectOptions{VersionID: results[i].VersionID} reader, err := c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "error during GET object", err) return } statInfo, err := reader.Stat() if err != nil { logError(testName, function, args, startTime, "", "error during calling reader.Stat()", err) return } if statInfo.ETag != results[i].ETag { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err) return } if statInfo.LastModified.Unix() != results[i].LastModified.Unix() { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err) return } if statInfo.Size != results[i].Size { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err) return } tmpBuffer := bytes.NewBuffer([]byte{}) _, err = io.Copy(tmpBuffer, reader) if err != nil { logError(testName, function, args, startTime, "", "unexpected io.Copy()", err) return } if !bytes.Equal(tmpBuffer.Bytes(), buffers[i]) { logError(testName, function, args, startTime, "", "unexpected content of GetObject()", err) return } } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testPutObjectWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName const n = 10 // Read input... // Save the data concurrently. var wg sync.WaitGroup wg.Add(n) buffers := make([][]byte, n) var errs [n]error for i := 0; i < n; i++ { r := newRandomReader(int64((1<<20)*i+i), int64(i)) buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return } buffers[i] = buf go func(i int) { defer wg.Done() _, errs[i] = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{PartSize: 5 << 20}) }(i) } wg.Wait() for _, err := range errs { if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } } objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var results []minio.ObjectInfo for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", info.Err) return } results = append(results, info) } if len(results) != n { logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) return } sort.Slice(results, func(i, j int) bool { return results[i].Size < results[j].Size }) sort.Slice(buffers, func(i, j int) bool { return len(buffers[i]) < len(buffers[j]) }) for i := 0; i < len(results); i++ { opts := minio.GetObjectOptions{VersionID: results[i].VersionID} reader, err := c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "error during GET object", err) return } statInfo, err := reader.Stat() if err != nil { logError(testName, function, args, startTime, "", "error during calling reader.Stat()", err) return } if statInfo.ETag != results[i].ETag { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err) return } if statInfo.LastModified.Unix() != results[i].LastModified.Unix() { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err) return } if statInfo.Size != results[i].Size { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err) return } tmpBuffer := bytes.NewBuffer([]byte{}) _, err = io.Copy(tmpBuffer, reader) if err != nil { logError(testName, function, args, startTime, "", "unexpected io.Copy()", err) return } if !bytes.Equal(tmpBuffer.Bytes(), buffers[i]) { logError(testName, function, args, startTime, "", "unexpected content of GetObject()", err) return } } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testListMultipartUpload() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } core := minio.Core{Client: c} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. ctx := context.Background() err = c.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer func() { if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) } }() objName := "prefix/objectName" want := minio.ListMultipartUploadsResult{ Bucket: bucketName, KeyMarker: "", UploadIDMarker: "", NextKeyMarker: "", NextUploadIDMarker: "", EncodingType: "url", MaxUploads: 1000, IsTruncated: false, Prefix: "prefix/objectName", Delimiter: "/", CommonPrefixes: nil, } for i := 0; i < 5; i++ { uid, err := core.NewMultipartUpload(ctx, bucketName, objName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload failed", err) return } want.Uploads = append(want.Uploads, minio.ObjectMultipartInfo{ Initiated: time.Time{}, StorageClass: "", Key: objName, Size: 0, UploadID: uid, Err: nil, }) for j := 0; j < 5; j++ { cmpGot := func(call string, got minio.ListMultipartUploadsResult) bool { for i := range got.Uploads { got.Uploads[i].Initiated = time.Time{} } if !reflect.DeepEqual(want, got) { err := fmt.Errorf("want: %#v\ngot : %#v", want, got) logError(testName, function, args, startTime, "", call+" failed", err) } return true } got, err := core.ListMultipartUploads(ctx, bucketName, objName, "", "", "/", 1000) if err != nil { logError(testName, function, args, startTime, "", "ListMultipartUploads failed", err) return } if !cmpGot("ListMultipartUploads-prefix", got) { return } got, err = core.ListMultipartUploads(ctx, bucketName, objName, objName, "", "/", 1000) got.KeyMarker = "" if err != nil { logError(testName, function, args, startTime, "", "ListMultipartUploads failed", err) return } if !cmpGot("ListMultipartUploads-marker", got) { return } } if i > 2 { err = core.AbortMultipartUpload(ctx, bucketName, objName, uid) if err != nil { logError(testName, function, args, startTime, "", "AbortMultipartUpload failed", err) return } want.Uploads = want.Uploads[:len(want.Uploads)-1] } } for _, up := range want.Uploads { err = core.AbortMultipartUpload(ctx, bucketName, objName, up.UploadID) if err != nil { logError(testName, function, args, startTime, "", "AbortMultipartUpload failed", err) return } } logSuccess(testName, function, args, startTime) } func testCopyObjectWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName testFiles := []string{"datafile-1-b", "datafile-10-kB"} for _, testFile := range testFiles { r := getDataReader(testFile) buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return } r.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } } objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var infos []minio.ObjectInfo for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } infos = append(infos, info) } sort.Slice(infos, func(i, j int) bool { return infos[i].Size < infos[j].Size }) reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{VersionID: infos[0].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "GetObject of the oldest version content failed", err) return } oldestContent, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err) return } // Copy Source srcOpts := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, VersionID: infos[0].VersionID, } args["src"] = srcOpts dstOpts := minio.CopyDestOptions{ Bucket: bucketName, Object: objectName + "-copy", } args["dst"] = dstOpts // Perform the Copy if _, err = c.CopyObject(context.Background(), dstOpts, srcOpts); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Destination object readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer readerCopy.Close() newestContent, err := io.ReadAll(readerCopy) if err != nil { logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err) return } if len(newestContent) == 0 || !bytes.Equal(oldestContent, newestContent) { logError(testName, function, args, startTime, "", "Unexpected destination object content", err) return } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testConcurrentCopyObjectWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName testFiles := []string{"datafile-10-kB"} for _, testFile := range testFiles { r := getDataReader(testFile) buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return } r.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } } objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var infos []minio.ObjectInfo for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } infos = append(infos, info) } sort.Slice(infos, func(i, j int) bool { return infos[i].Size < infos[j].Size }) reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{VersionID: infos[0].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "GetObject of the oldest version content failed", err) return } oldestContent, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err) return } // Copy Source srcOpts := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, VersionID: infos[0].VersionID, } args["src"] = srcOpts dstOpts := minio.CopyDestOptions{ Bucket: bucketName, Object: objectName + "-copy", } args["dst"] = dstOpts // Perform the Copy concurrently const n = 10 var wg sync.WaitGroup wg.Add(n) var errs [n]error for i := 0; i < n; i++ { go func(i int) { defer wg.Done() _, errs[i] = c.CopyObject(context.Background(), dstOpts, srcOpts) }(i) } wg.Wait() for _, err := range errs { if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } } objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: false, Prefix: dstOpts.Object}) infos = []minio.ObjectInfo{} for info := range objectsInfo { // Destination object readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{VersionID: info.VersionID}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer readerCopy.Close() newestContent, err := io.ReadAll(readerCopy) if err != nil { logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err) return } if len(newestContent) == 0 || !bytes.Equal(oldestContent, newestContent) { logError(testName, function, args, startTime, "", "Unexpected destination object content", err) return } infos = append(infos, info) } if len(infos) != n { logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) return } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testComposeObjectWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ComposeObject()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // var testFiles = []string{"datafile-5-MB", "datafile-10-kB"} testFiles := []string{"datafile-5-MB", "datafile-10-kB"} var testFilesBytes [][]byte for _, testFile := range testFiles { r := getDataReader(testFile) buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return } r.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } testFilesBytes = append(testFilesBytes, buf) } objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var results []minio.ObjectInfo for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } results = append(results, info) } sort.SliceStable(results, func(i, j int) bool { return results[i].Size > results[j].Size }) // Source objects to concatenate. We also specify decryption // key for each src1 := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, VersionID: results[0].VersionID, } src2 := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, VersionID: results[1].VersionID, } dst := minio.CopyDestOptions{ Bucket: bucketName, Object: objectName + "-copy", } _, err = c.ComposeObject(context.Background(), dst, src1, src2) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return } // Destination object readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject of the copy object failed", err) return } defer readerCopy.Close() copyContentBytes, err := io.ReadAll(readerCopy) if err != nil { logError(testName, function, args, startTime, "", "Reading from the copy object reader failed", err) return } var expectedContent []byte for _, fileBytes := range testFilesBytes { expectedContent = append(expectedContent, fileBytes...) } if len(copyContentBytes) == 0 || !bytes.Equal(copyContentBytes, expectedContent) { logError(testName, function, args, startTime, "", "Unexpected destination object content", err) return } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testRemoveObjectWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "DeleteObject()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var version minio.ObjectInfo for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } version = info break } err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{VersionID: version.VersionID}) if err != nil { logError(testName, function, args, startTime, "", "DeleteObject failed", err) return } objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) for range objectsInfo { logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err) return } // test delete marker version id is non-null _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // create delete marker err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "DeleteObject failed", err) return } objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) idx := 0 for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } if idx == 0 { if !info.IsDeleteMarker { logError(testName, function, args, startTime, "", "Unexpected error - expected delete marker to have been created", err) return } if info.VersionID == "" { logError(testName, function, args, startTime, "", "Unexpected error - expected delete marker to be versioned", err) return } } idx++ } defer cleanupBucket(bucketName, c) logSuccess(testName, function, args, startTime) } func testRemoveObjectsWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "DeleteObjects()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } objectsVersions := make(chan minio.ObjectInfo) go func() { objectsVersionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) for info := range objectsVersionsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } objectsVersions <- info } close(objectsVersions) }() removeErrors := c.RemoveObjects(context.Background(), bucketName, objectsVersions, minio.RemoveObjectsOptions{}) if err != nil { logError(testName, function, args, startTime, "", "DeleteObjects call failed", err) return } for e := range removeErrors { if e.Err != nil { logError(testName, function, args, startTime, "", "Single delete operation failed", err) return } } objectsVersionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) for range objectsVersionsInfo { logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err) return } err = c.RemoveBucket(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testObjectTaggingWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "{Get,Set,Remove}ObjectTagging()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName for _, file := range []string{"datafile-1-b", "datafile-10-kB"} { _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader(file), int64(dataFileMap[file]), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } } versionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var versions []minio.ObjectInfo for info := range versionsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } versions = append(versions, info) } sort.SliceStable(versions, func(i, j int) bool { return versions[i].Size < versions[j].Size }) tagsV1 := map[string]string{"key1": "val1"} t1, err := tags.MapToObjectTags(tagsV1) if err != nil { logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err) return } err = c.PutObjectTagging(context.Background(), bucketName, objectName, t1, minio.PutObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err) return } tagsV2 := map[string]string{"key2": "val2"} t2, err := tags.MapToObjectTags(tagsV2) if err != nil { logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err) return } err = c.PutObjectTagging(context.Background(), bucketName, objectName, t2, minio.PutObjectTaggingOptions{VersionID: versions[1].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectTagging (2) failed", err) return } tagsEqual := func(tags1, tags2 map[string]string) bool { for k1, v1 := range tags1 { v2, found := tags2[k1] if found { if v1 != v2 { return false } } } return true } gotTagsV1, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "GetObjectTagging failed", err) return } if !tagsEqual(t1.ToMap(), gotTagsV1.ToMap()) { logError(testName, function, args, startTime, "", "Unexpected tags content (1)", err) return } gotTagsV2, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObjectTaggingContext failed", err) return } if !tagsEqual(t2.ToMap(), gotTagsV2.ToMap()) { logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err) return } err = c.RemoveObjectTagging(context.Background(), bucketName, objectName, minio.RemoveObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectTagging (2) failed", err) return } emptyTags, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "GetObjectTagging failed", err) return } if len(emptyTags.ToMap()) != 0 { logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err) return } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Test PutObject with custom checksums. func testPutObjectWithChecksums() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) tests := []struct { cs minio.ChecksumType }{ {cs: minio.ChecksumCRC32C}, {cs: minio.ChecksumCRC32}, {cs: minio.ChecksumSHA1}, {cs: minio.ChecksumSHA256}, {cs: minio.ChecksumCRC64NVME}, } for _, test := range tests { if os.Getenv("MINT_NO_FULL_OBJECT") != "" && test.cs.FullObjectRequested() { continue } bufSize := dataFileMap["datafile-10-kB"] // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName cmpChecksum := func(got, want string) { if want != got { logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) return } } meta := map[string]string{} reader := getDataReader("datafile-10-kB") b, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } h := test.cs.Hasher() h.Reset() if test.cs.IsSet() { meta["x-amz-checksum-algorithm"] = test.cs.String() } // Test with a bad CRC - we haven't called h.Write(b), so this is a checksum of empty data meta[test.cs.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil)) args["metadata"] = meta args["range"] = "false" args["checksum"] = test.cs.String() resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ DisableMultipart: true, UserMetadata: meta, }) if err == nil { logError(testName, function, args, startTime, "", "PutObject did not fail on wrong CRC", err) return } // Set correct CRC. h.Write(b) meta[test.cs.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil)) reader.Close() resp, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ DisableMultipart: true, DisableContentSha256: true, UserMetadata: meta, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"]) if resp.ChecksumMode != minio.ChecksumFullObjectMode.String() { logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", resp.ChecksumMode, minio.ChecksumFullObjectMode.String())) } // Read the data back gopts := minio.GetObjectOptions{Checksum: true} r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } cmpChecksum(st.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"]) if st.ChecksumMode != minio.ChecksumFullObjectMode.String() { logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", st.ChecksumMode, minio.ChecksumFullObjectMode.String())) } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", "Object Close failed", err) return } if err := r.Close(); err == nil { logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err) return } args["range"] = "true" err = gopts.SetRange(100, 1000) if err != nil { logError(testName, function, args, startTime, "", "SetRange failed", err) return } r, err = c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } b, err = io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } st, err = r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } // Range requests should return empty checksums... cmpChecksum(st.ChecksumSHA256, "") cmpChecksum(st.ChecksumSHA1, "") cmpChecksum(st.ChecksumCRC32, "") cmpChecksum(st.ChecksumCRC32C, "") cmpChecksum(st.ChecksumCRC64NVME, "") delete(args, "range") delete(args, "metadata") logSuccess(testName, function, args, startTime) } } // Test PutObject with custom checksums. func testPutObjectWithTrailingChecksums() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader,size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress, TrailChecksum: xxx}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) tests := []struct { cs minio.ChecksumType }{ {cs: minio.ChecksumCRC64NVME}, {cs: minio.ChecksumCRC32C}, {cs: minio.ChecksumCRC32}, {cs: minio.ChecksumSHA1}, {cs: minio.ChecksumSHA256}, } for _, test := range tests { if os.Getenv("MINT_NO_FULL_OBJECT") != "" && test.cs.FullObjectRequested() { continue } function := "PutObject(bucketName, objectName, reader,size, opts)" bufSize := dataFileMap["datafile-10-kB"] // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName cmpChecksum := func(got, want string) { if want != got { logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) return } } meta := map[string]string{} reader := getDataReader("datafile-10-kB") b, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } h := test.cs.Hasher() h.Reset() // Test with Wrong CRC. args["metadata"] = meta args["range"] = "false" args["checksum"] = test.cs.String() resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ DisableMultipart: true, DisableContentSha256: true, UserMetadata: meta, Checksum: test.cs, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } h.Write(b) meta[test.cs.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil)) cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"]) // Read the data back gopts := minio.GetObjectOptions{Checksum: true} function = "GetObject(...)" r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } cmpChecksum(st.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"]) if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", "Object Close failed", err) return } if err := r.Close(); err == nil { logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err) return } function = "GetObject( Range...)" args["range"] = "true" err = gopts.SetRange(100, 1000) if err != nil { logError(testName, function, args, startTime, "", "SetRange failed", err) return } r, err = c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } b, err = io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } st, err = r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } // Range requests should return empty checksums... cmpChecksum(st.ChecksumSHA256, "") cmpChecksum(st.ChecksumSHA1, "") cmpChecksum(st.ChecksumCRC32, "") cmpChecksum(st.ChecksumCRC32C, "") cmpChecksum(st.ChecksumCRC64NVME, "") function = "GetObjectAttributes(...)" s, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, minio.ObjectAttributesOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObjectAttributes failed", err) return } cmpChecksum(s.Checksum.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(s.Checksum.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(s.Checksum.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(s.Checksum.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) delete(args, "range") delete(args, "metadata") logSuccess(testName, function, args, startTime) } } // Test PutObject with custom checksums. func testPutMultipartObjectWithChecksums() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader,size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{UserMetadata: metadata, Trailing: true}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } hashMultiPart := func(b []byte, partSize int, cs minio.ChecksumType) string { r := bytes.NewReader(b) hasher := cs.Hasher() if cs.FullObjectRequested() { partSize = len(b) } tmp := make([]byte, partSize) parts := 0 var all []byte for { n, err := io.ReadFull(r, tmp) if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF { logError(testName, function, args, startTime, "", "Calc crc failed", err) } if n == 0 { break } parts++ hasher.Reset() hasher.Write(tmp[:n]) all = append(all, hasher.Sum(nil)...) if err != nil { break } } if parts == 1 { return base64.StdEncoding.EncodeToString(hasher.Sum(nil)) } hasher.Reset() hasher.Write(all) return fmt.Sprintf("%s-%d", base64.StdEncoding.EncodeToString(hasher.Sum(nil)), parts) } defer cleanupBucket(bucketName, c) tests := []struct { cs minio.ChecksumType }{ {cs: minio.ChecksumFullObjectCRC32}, {cs: minio.ChecksumFullObjectCRC32C}, {cs: minio.ChecksumCRC64NVME}, {cs: minio.ChecksumCRC32C}, {cs: minio.ChecksumCRC32}, {cs: minio.ChecksumSHA1}, {cs: minio.ChecksumSHA256}, } for _, test := range tests { if os.Getenv("MINT_NO_FULL_OBJECT") != "" && test.cs.FullObjectRequested() { continue } args["section"] = "prep" bufSize := dataFileMap["datafile-129-MB"] // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName args["checksum"] = test.cs.String() cmpChecksum := func(got, want string) { if want != got { logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) // fmt.Printf("want %s, got %s\n", want, got) return } } const partSize = 10 << 20 reader := getDataReader("datafile-129-MB") b, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } reader.Close() h := test.cs.Hasher() h.Reset() // wantChksm might be the full object checksum or the multipart checksum, depending on the test.cs type. wantChksm := hashMultiPart(b, partSize, test.cs) // wantFullObjectChksm is always the full object checksum that is returned after CopyObject. wantFullObjectChksm := hashMultiPart(b, len(b), test.cs) rd := bytes.NewReader(b) cs := test.cs // Set correct CRC. args["section"] = "PutObject" resp, err := c.PutObject(context.Background(), bucketName, objectName, rd, int64(bufSize), minio.PutObjectOptions{ DisableContentSha256: true, DisableMultipart: false, UserMetadata: nil, PartSize: partSize, Checksum: cs, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } switch test.cs.Base() { case minio.ChecksumCRC32C: cmpChecksum(resp.ChecksumCRC32C, wantChksm) case minio.ChecksumCRC32: cmpChecksum(resp.ChecksumCRC32, wantChksm) case minio.ChecksumSHA1: cmpChecksum(resp.ChecksumSHA1, wantChksm) case minio.ChecksumSHA256: cmpChecksum(resp.ChecksumSHA256, wantChksm) case minio.ChecksumCRC64NVME: cmpChecksum(resp.ChecksumCRC64NVME, wantChksm) } args["section"] = "HeadObject" st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{Checksum: true}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } switch test.cs.Base() { case minio.ChecksumCRC32C: cmpChecksum(st.ChecksumCRC32C, wantChksm) case minio.ChecksumCRC32: cmpChecksum(st.ChecksumCRC32, wantChksm) case minio.ChecksumSHA1: cmpChecksum(st.ChecksumSHA1, wantChksm) case minio.ChecksumSHA256: cmpChecksum(st.ChecksumSHA256, wantChksm) case minio.ChecksumCRC64NVME: cmpChecksum(st.ChecksumCRC64NVME, wantChksm) } // Use the CopyObject API to make a copy, in the case it was a composite checksum, // it will change because the copy is no longer a multipart object. S3 returns the checksum // of the full object when HeadObject is called on the copy. args["section"] = "CopyObject" objectCopyName := objectName + "-copy" _, err = c.CopyObject(context.Background(), minio.CopyDestOptions{ Bucket: bucketName, Object: objectCopyName, }, minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, }) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } args["section"] = "HeadObject-Copy" st, err = c.StatObject(context.Background(), bucketName, objectCopyName, minio.StatObjectOptions{Checksum: true}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } switch test.cs.Base() { case minio.ChecksumCRC32C: cmpChecksum(st.ChecksumCRC32C, wantFullObjectChksm) case minio.ChecksumCRC32: cmpChecksum(st.ChecksumCRC32, wantFullObjectChksm) case minio.ChecksumSHA1: cmpChecksum(st.ChecksumSHA1, wantFullObjectChksm) case minio.ChecksumSHA256: cmpChecksum(st.ChecksumSHA256, wantFullObjectChksm) case minio.ChecksumCRC64NVME: cmpChecksum(st.ChecksumCRC64NVME, wantFullObjectChksm) } args["section"] = "GetObjectAttributes" s, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, minio.ObjectAttributesOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObjectAttributes failed", err) return } if strings.ContainsRune(wantChksm, '-') { wantChksm = wantChksm[:strings.IndexByte(wantChksm, '-')] } switch test.cs { // Full Object CRC does not return anything with GetObjectAttributes case minio.ChecksumCRC32C: cmpChecksum(s.Checksum.ChecksumCRC32C, wantChksm) case minio.ChecksumCRC32: cmpChecksum(s.Checksum.ChecksumCRC32, wantChksm) case minio.ChecksumSHA1: cmpChecksum(s.Checksum.ChecksumSHA1, wantChksm) case minio.ChecksumSHA256: cmpChecksum(s.Checksum.ChecksumSHA256, wantChksm) } // Read the data back gopts := minio.GetObjectOptions{Checksum: true} gopts.PartNumber = 2 // We cannot use StatObject, since it ignores partnumber. args["section"] = "GetObject-Part" r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } io.Copy(io.Discard, r) st, err = r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } // Test part 2 checksum... h.Reset() h.Write(b[partSize : 2*partSize]) wantChksm = base64.StdEncoding.EncodeToString(h.Sum(nil)) switch test.cs { // Full Object CRC does not return any part CRC for whatever reason. case minio.ChecksumCRC32C: cmpChecksum(st.ChecksumCRC32C, wantChksm) case minio.ChecksumCRC32: cmpChecksum(st.ChecksumCRC32, wantChksm) case minio.ChecksumSHA1: cmpChecksum(st.ChecksumSHA1, wantChksm) case minio.ChecksumSHA256: cmpChecksum(st.ChecksumSHA256, wantChksm) case minio.ChecksumCRC64NVME: // AWS doesn't return part checksum, but may in the future. if st.ChecksumCRC64NVME != "" { cmpChecksum(st.ChecksumCRC64NVME, wantChksm) } } delete(args, "metadata") delete(args, "section") logSuccess(testName, function, args, startTime) } } // Test PutObject with trailing checksums. func testTrailingChecksums() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader,size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } hashMultiPart := func(b []byte, partSize int, hasher hash.Hash) (oparts []minio.ObjectPart) { r := bytes.NewReader(b) tmp := make([]byte, partSize) parts := 0 for { n, err := io.ReadFull(r, tmp) if err != nil && err != io.ErrUnexpectedEOF { logError(testName, function, args, startTime, "", "Calc crc failed", err) } if n == 0 { break } parts++ hasher.Reset() hasher.Write(tmp[:n]) oparts = append(oparts, minio.ObjectPart{ PartNumber: parts, Size: int64(n), ChecksumCRC32C: base64.StdEncoding.EncodeToString(hasher.Sum(nil)), }) if err != nil { break } } return oparts } defer cleanupBucket(bucketName, c) tests := []struct { header string hasher hash.Hash // Checksum values ChecksumCRC32 string ChecksumCRC32C string ChecksumSHA1 string ChecksumSHA256 string PO minio.PutObjectOptions }{ // Currently there is no way to override the checksum type. { header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), ChecksumCRC32C: "set", PO: minio.PutObjectOptions{ DisableContentSha256: true, DisableMultipart: false, UserMetadata: nil, PartSize: 5 << 20, Checksum: minio.ChecksumFullObjectCRC32C, }, }, { header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), ChecksumCRC32C: "set", PO: minio.PutObjectOptions{ DisableContentSha256: true, DisableMultipart: false, UserMetadata: nil, PartSize: 6_645_654, // Rather arbitrary size Checksum: minio.ChecksumFullObjectCRC32C, }, }, { header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), ChecksumCRC32C: "set", PO: minio.PutObjectOptions{ DisableContentSha256: false, DisableMultipart: false, UserMetadata: nil, PartSize: 5 << 20, Checksum: minio.ChecksumFullObjectCRC32C, }, }, { header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), ChecksumCRC32C: "set", PO: minio.PutObjectOptions{ DisableContentSha256: false, DisableMultipart: false, UserMetadata: nil, PartSize: 6_645_654, // Rather arbitrary size Checksum: minio.ChecksumFullObjectCRC32C, }, }, } for _, test := range tests { bufSize := dataFileMap["datafile-11-MB"] // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName cmpChecksum := func(got, want string) { if want != got { logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %q, got %q", want, got)) return } } reader := getDataReader("datafile-11-MB") b, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } reader.Close() h := test.hasher h.Reset() parts := hashMultiPart(b, int(test.PO.PartSize), test.hasher) cksum, err := minio.ChecksumFullObjectCRC32C.FullObjectChecksum(parts) if err != nil { logError(testName, function, args, startTime, "", "checksum calculation failed", err) return } test.ChecksumCRC32C = cksum.Encoded() // Set correct CRC. resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), test.PO) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // c.TraceOff() cmpChecksum(resp.ChecksumSHA256, test.ChecksumSHA256) cmpChecksum(resp.ChecksumSHA1, test.ChecksumSHA1) cmpChecksum(resp.ChecksumCRC32, test.ChecksumCRC32) cmpChecksum(resp.ChecksumCRC32C, test.ChecksumCRC32C) // Read the data back gopts := minio.GetObjectOptions{Checksum: true} gopts.PartNumber = 2 // We cannot use StatObject, since it ignores partnumber. r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } io.Copy(io.Discard, r) st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } // Test part 2 checksum... h.Reset() p2 := b[test.PO.PartSize:] if len(p2) > int(test.PO.PartSize) { p2 = p2[:test.PO.PartSize] } h.Write(p2) got := base64.StdEncoding.EncodeToString(h.Sum(nil)) if test.ChecksumSHA256 != "" { cmpChecksum(st.ChecksumSHA256, got) } if test.ChecksumSHA1 != "" { cmpChecksum(st.ChecksumSHA1, got) } if test.ChecksumCRC32 != "" { cmpChecksum(st.ChecksumCRC32, got) } if test.ChecksumCRC32C != "" { cmpChecksum(st.ChecksumCRC32C, got) } delete(args, "metadata") logSuccess(testName, function, args, startTime) } } // Test PutObject with custom checksums. func testPutObjectWithAutomaticChecksums() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader,size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) tests := []struct { header string hasher hash.Hash // Checksum values ChecksumCRC32 string ChecksumCRC32C string ChecksumSHA1 string ChecksumSHA256 string }{ // Built-in will only add crc32c, when no MD5 nor SHA256. {header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli))}, } // defer c.TraceOff() for i, test := range tests { bufSize := dataFileMap["datafile-10-kB"] // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName cmpChecksum := func(got, want string) { if want != got { logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) return } } meta := map[string]string{} reader := getDataReader("datafile-10-kB") b, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } h := test.hasher h.Reset() h.Write(b) meta[test.header] = base64.StdEncoding.EncodeToString(h.Sum(nil)) args["metadata"] = meta resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ DisableMultipart: true, UserMetadata: nil, DisableContentSha256: true, SendContentMd5: false, }) if err == nil { if i == 0 && resp.ChecksumCRC32C == "" { logIgnored(testName, function, args, startTime, "Checksums does not appear to be supported by backend") return } } else { logError(testName, function, args, startTime, "", "PutObject failed", err) return } cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) // Usually this will be the same as above, since we skip automatic checksum when SHA256 content is sent. // When/if we add a checksum control to PutObjectOptions this will make more sense. resp, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ DisableMultipart: true, UserMetadata: nil, DisableContentSha256: false, SendContentMd5: false, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // The checksum will not be enabled on HTTP, since it uses SHA256 blocks. if mustParseBool(os.Getenv(enableHTTPS)) { cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) } // Set SHA256 header manually sh256 := sha256.Sum256(b) meta = map[string]string{"x-amz-checksum-sha256": base64.StdEncoding.EncodeToString(sh256[:])} args["metadata"] = meta resp, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ DisableMultipart: true, UserMetadata: meta, DisableContentSha256: true, SendContentMd5: false, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) delete(args, "metadata") } logSuccess(testName, function, args, startTime) } func testGetObjectAttributes() { startTime := time.Now() testName := getFuncName() function := "GetObjectAttributes(ctx, bucketName, objectName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.ObjectAttributesOptions{}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket( context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}, ) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } bucketNameV := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-versioned-") args["bucketName"] = bucketNameV err = c.MakeBucket( context.Background(), bucketNameV, minio.MakeBucketOptions{Region: "us-east-1"}, ) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketNameV) if err != nil { logError(testName, function, args, startTime, "", "Unable to enable versioning", err) return } defer cleanupBucket(bucketName, c) defer cleanupVersionedBucket(bucketNameV, c) testFiles := make(map[string]*objectAttributesNewObject) testFiles["file1"] = &objectAttributesNewObject{ Object: "file1", ObjectReaderType: "datafile-1.03-MB", Bucket: bucketNameV, ContentType: "custom/contenttype", SendContentMd5: false, } testFiles["file2"] = &objectAttributesNewObject{ Object: "file2", ObjectReaderType: "datafile-129-MB", Bucket: bucketName, ContentType: "custom/contenttype", SendContentMd5: false, } for i, v := range testFiles { bufSize := dataFileMap[v.ObjectReaderType] reader := getDataReader(v.ObjectReaderType) args["objectName"] = v.Object testFiles[i].UploadInfo, err = c.PutObject(context.Background(), v.Bucket, v.Object, reader, int64(bufSize), minio.PutObjectOptions{ ContentType: v.ContentType, SendContentMd5: v.SendContentMd5, Checksum: minio.ChecksumCRC32C, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } } testTable := make(map[string]objectAttributesTableTest) testTable["none-versioned"] = objectAttributesTableTest{ opts: minio.ObjectAttributesOptions{}, test: objectAttributesTestOptions{ TestFileName: "file2", StorageClass: "STANDARD", HasFullChecksum: true, HasPartChecksums: true, HasParts: true, }, } testTable["0-to-0-marker"] = objectAttributesTableTest{ opts: minio.ObjectAttributesOptions{ PartNumberMarker: 0, MaxParts: 0, }, test: objectAttributesTestOptions{ TestFileName: "file2", StorageClass: "STANDARD", HasFullChecksum: true, HasPartChecksums: true, HasParts: true, }, } testTable["0-marker-to-max"] = objectAttributesTableTest{ opts: minio.ObjectAttributesOptions{ PartNumberMarker: 0, MaxParts: 10000, }, test: objectAttributesTestOptions{ TestFileName: "file2", StorageClass: "STANDARD", HasFullChecksum: true, HasPartChecksums: true, HasParts: true, }, } testTable["0-to-1-marker"] = objectAttributesTableTest{ opts: minio.ObjectAttributesOptions{ PartNumberMarker: 0, MaxParts: 1, }, test: objectAttributesTestOptions{ TestFileName: "file2", StorageClass: "STANDARD", HasFullChecksum: true, HasPartChecksums: true, HasParts: true, }, } testTable["7-to-6-marker"] = objectAttributesTableTest{ opts: minio.ObjectAttributesOptions{ PartNumberMarker: 7, MaxParts: 6, }, test: objectAttributesTestOptions{ TestFileName: "file2", StorageClass: "STANDARD", HasFullChecksum: true, HasPartChecksums: true, HasParts: true, }, } testTable["versioned"] = objectAttributesTableTest{ opts: minio.ObjectAttributesOptions{}, test: objectAttributesTestOptions{ TestFileName: "file1", StorageClass: "STANDARD", HasFullChecksum: true, }, } for i, v := range testTable { tf, ok := testFiles[v.test.TestFileName] if !ok { continue } args["objectName"] = tf.Object args["bucketName"] = tf.Bucket if tf.UploadInfo.VersionID != "" { v.opts.VersionID = tf.UploadInfo.VersionID } s, err := c.GetObjectAttributes(context.Background(), tf.Bucket, tf.Object, v.opts) if err != nil { logError(testName, function, args, startTime, "", "GetObjectAttributes failed", err) return } v.test.NumberOfParts = s.ObjectParts.PartsCount v.test.ETag = tf.UploadInfo.ETag v.test.ObjectSize = int(tf.UploadInfo.Size) err = validateObjectAttributeRequest(s, &v.opts, &v.test) if err != nil { logError(testName, function, args, startTime, "", "Validating GetObjectsAttributes response failed, table test: "+i, err) return } } logSuccess(testName, function, args, startTime) } func testGetObjectAttributesSSECEncryption() { startTime := time.Now() testName := getFuncName() function := "GetObjectAttributes(ctx, bucketName, objectName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.ObjectAttributesOptions{}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket( context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}, ) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) objectName := "encrypted-object" args["objectName"] = objectName bufSize := dataFileMap["datafile-11-MB"] reader := getDataReader("datafile-11-MB") sse := encrypt.DefaultPBKDF([]byte("word1 word2 word3 word4"), []byte(bucketName+objectName)) info, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ ContentType: "content/custom", SendContentMd5: false, ServerSideEncryption: sse, PartSize: uint64(bufSize) / 2, Checksum: minio.ChecksumCRC32C, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } opts := minio.ObjectAttributesOptions{ ServerSideEncryption: sse, } attr, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "GetObjectAttributes with empty bucket name should have failed", nil) return } err = validateObjectAttributeRequest(attr, &opts, &objectAttributesTestOptions{ TestFileName: info.Key, ETag: info.ETag, NumberOfParts: 2, ObjectSize: int(info.Size), HasFullChecksum: true, HasParts: true, HasPartChecksums: true, }) if err != nil { logError(testName, function, args, startTime, "", "Validating GetObjectsAttributes response failed", err) return } logSuccess(testName, function, args, startTime) } func testGetObjectAttributesErrorCases() { startTime := time.Now() testName := getFuncName() function := "GetObjectAttributes(ctx, bucketName, objectName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.ObjectAttributesOptions{}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } unknownBucket := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-bucket-") unknownObject := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-object-") _, err = c.GetObjectAttributes(context.Background(), unknownBucket, unknownObject, minio.ObjectAttributesOptions{}) if err == nil { logError(testName, function, args, startTime, "", "GetObjectAttributes failed", nil) return } errorResponse := err.(minio.ErrorResponse) if errorResponse.Code != minio.NoSuchBucket { logError(testName, function, args, startTime, "", "Invalid error code, expected NoSuchBucket but got "+errorResponse.Code, nil) return } bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket( context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}, ) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } bucketNameV := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-versioned-") args["bucketName"] = bucketNameV err = c.MakeBucket( context.Background(), bucketNameV, minio.MakeBucketOptions{Region: "us-east-1"}, ) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketNameV) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) defer cleanupVersionedBucket(bucketNameV, c) _, err = c.GetObjectAttributes(context.Background(), bucketName, unknownObject, minio.ObjectAttributesOptions{}) if err == nil { logError(testName, function, args, startTime, "", "GetObjectAttributes failed", nil) return } errorResponse = err.(minio.ErrorResponse) if errorResponse.Code != minio.NoSuchKey { logError(testName, function, args, startTime, "", "Invalid error code, expected "+minio.NoSuchKey+" but got "+errorResponse.Code, nil) return } _, err = c.GetObjectAttributes(context.Background(), bucketName, "", minio.ObjectAttributesOptions{}) if err == nil { logError(testName, function, args, startTime, "", "GetObjectAttributes with empty object name should have failed", nil) return } _, err = c.GetObjectAttributes(context.Background(), "", unknownObject, minio.ObjectAttributesOptions{}) if err == nil { logError(testName, function, args, startTime, "", "GetObjectAttributes with empty bucket name should have failed", nil) return } _, err = c.GetObjectAttributes(context.Background(), bucketNameV, unknownObject, minio.ObjectAttributesOptions{ VersionID: uuid.NewString(), }) if err == nil { logError(testName, function, args, startTime, "", "GetObjectAttributes with empty bucket name should have failed", nil) return } errorResponse = err.(minio.ErrorResponse) if errorResponse.Code != minio.NoSuchVersion { logError(testName, function, args, startTime, "", "Invalid error code, expected "+minio.NoSuchVersion+" but got "+errorResponse.Code, nil) return } logSuccess(testName, function, args, startTime) } type objectAttributesNewObject struct { Object string ObjectReaderType string Bucket string ContentType string SendContentMd5 bool UploadInfo minio.UploadInfo } type objectAttributesTableTest struct { opts minio.ObjectAttributesOptions test objectAttributesTestOptions } type objectAttributesTestOptions struct { TestFileName string ETag string NumberOfParts int StorageClass string ObjectSize int HasPartChecksums bool HasFullChecksum bool HasParts bool } func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.ObjectAttributesOptions, test *objectAttributesTestOptions) (err error) { if opts.VersionID != "" { if OA.VersionID != opts.VersionID { err = fmt.Errorf("Expected versionId %s but got versionId %s", opts.VersionID, OA.VersionID) return err } } partsMissingChecksum := false foundPartChecksum := false for _, v := range OA.ObjectParts.Parts { checksumFound := false if v.ChecksumSHA256 != "" { checksumFound = true } else if v.ChecksumSHA1 != "" { checksumFound = true } else if v.ChecksumCRC32 != "" { checksumFound = true } else if v.ChecksumCRC32C != "" { checksumFound = true } if !checksumFound { partsMissingChecksum = true } else { foundPartChecksum = true } } if test.HasPartChecksums { if partsMissingChecksum { err = fmt.Errorf("One or all parts were missing a checksum") return err } } else { if foundPartChecksum { err = fmt.Errorf("Did not expect ObjectParts to have checksums but found one") return err } } hasFullObjectChecksum := (OA.Checksum.ChecksumCRC32 != "" || OA.Checksum.ChecksumCRC32C != "" || OA.Checksum.ChecksumSHA1 != "" || OA.Checksum.ChecksumSHA256 != "") if test.HasFullChecksum { if !hasFullObjectChecksum { err = fmt.Errorf("Full object checksum not found") return err } } else { if hasFullObjectChecksum { err = fmt.Errorf("Did not expect a full object checksum but we got one") return err } } if OA.ETag != test.ETag { err = fmt.Errorf("Etags do not match, got %s but expected %s", OA.ETag, test.ETag) return err } if test.HasParts { if len(OA.ObjectParts.Parts) < 1 { err = fmt.Errorf("Was expecting ObjectParts but none were present") return err } } if OA.StorageClass == "" { err = fmt.Errorf("Was expecting a StorageClass but got none") return err } if OA.ObjectSize != test.ObjectSize { err = fmt.Errorf("Was expecting a ObjectSize but got none") return err } if test.HasParts { if opts.MaxParts == 0 { if len(OA.ObjectParts.Parts) != OA.ObjectParts.PartsCount { err = fmt.Errorf("expected %s parts but got %d", OA.ObjectParts.PartsCount, len(OA.ObjectParts.Parts)) return err } } else if (opts.MaxParts + opts.PartNumberMarker) > OA.ObjectParts.PartsCount { if len(OA.ObjectParts.Parts) != (OA.ObjectParts.PartsCount - opts.PartNumberMarker) { err = fmt.Errorf("expected %d parts but got %d", (OA.ObjectParts.PartsCount - opts.PartNumberMarker), len(OA.ObjectParts.Parts)) return err } } else if opts.MaxParts != 0 { if opts.MaxParts != len(OA.ObjectParts.Parts) { err = fmt.Errorf("expected %d parts but got %d", opts.MaxParts, len(OA.ObjectParts.Parts)) return err } } } if OA.ObjectParts.NextPartNumberMarker == OA.ObjectParts.PartsCount { if OA.ObjectParts.IsTruncated { err = fmt.Errorf("Expected ObjectParts to NOT be truncated, but it was") return err } } if OA.ObjectParts.NextPartNumberMarker != OA.ObjectParts.PartsCount { if !OA.ObjectParts.IsTruncated { err = fmt.Errorf("Expected ObjectParts to be truncated, but it was NOT") return err } } return err } // Test PutObject using a large data to trigger multipart readat func testPutObjectWithMetadata() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader,size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Object custom metadata customContentType := "custom/contenttype" args["metadata"] = map[string][]string{ "Content-Type": {customContentType}, "X-Amz-Meta-CustomKey": {"extra spaces in value"}, } _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ ContentType: customContentType, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) return } if st.ContentType != customContentType && st.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "ContentType does not match, expected "+customContentType+" got "+st.ContentType, err) return } if err := crcMatchesName(r, "datafile-129-MB"); err != nil { logError(testName, function, args, startTime, "", "data CRC check failed", err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", "Object Close failed", err) return } if err := r.Close(); err == nil { logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err) return } logSuccess(testName, function, args, startTime) } func testPutObjectWithContentLanguage() { // initialize logging params objectName := "test-object" startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": objectName, "size": -1, "opts": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) data := []byte{} _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(data), int64(0), minio.PutObjectOptions{ ContentLanguage: "en", }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if objInfo.Metadata.Get("Content-Language") != "en" { logError(testName, function, args, startTime, "", "Expected content-language 'en' doesn't match with StatObject return value", err) return } logSuccess(testName, function, args, startTime) } // Test put object with streaming signature. func testPutObjectStreaming() { // initialize logging params objectName := "test-object" startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader,size,opts)" args := map[string]interface{}{ "bucketName": "", "objectName": objectName, "size": -1, "opts": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Upload an object. sizes := []int64{0, 64*1024 - 1, 64 * 1024} for _, size := range sizes { data := newRandomReader(size, size) ui, err := c.PutObject(context.Background(), bucketName, objectName, data, int64(size), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) return } if ui.Size != size { logError(testName, function, args, startTime, "", "PutObjectStreaming result has unexpected size", nil) return } objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if objInfo.Size != size { logError(testName, function, args, startTime, "", "Unexpected size", err) return } } logSuccess(testName, function, args, startTime) } // Test PutObject with preconditions on non-existent objects func testPutObjectPreconditionOnNonExistent() { startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, size, opts) with preconditions" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{SetMatchETag/SetMatchETagExcept}", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Test 1: PutObject with SetMatchETag on non-existent object should fail objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "test-object-") args["objectName"] = objectName data := bytes.NewReader([]byte("test data")) opts := minio.PutObjectOptions{} opts.SetMatchETag("some-etag") _, err = c.PutObject(context.Background(), bucketName, objectName, data, int64(data.Len()), opts) if err == nil { logError(testName, function, args, startTime, "", "PutObject with SetMatchETag on non-existent object should have failed", nil) return } errResp := minio.ToErrorResponse(err) if errResp.Code != "NoSuchKey" { logError(testName, function, args, startTime, "", fmt.Sprintf("Expected NoSuchKey error (AWS standard for non-existent objects), got %s", errResp.Code), err) return } // Test 2: PutObject with SetMatchETagExcept (If-None-Match) on non-existent object should succeed objectName2 := randString(60, rand.NewSource(time.Now().UnixNano()), "test-object2-") args["objectName"] = objectName2 data2 := bytes.NewReader([]byte("test data 2")) opts2 := minio.PutObjectOptions{} opts2.SetMatchETagExcept("some-etag") _, err = c.PutObject(context.Background(), bucketName, objectName2, data2, int64(data2.Len()), opts2) if err != nil { logError(testName, function, args, startTime, "", "PutObject with SetMatchETagExcept (If-None-Match) on non-existent object should have succeeded", err) return } // Test 3: CompleteMultipartUpload with preconditions on non-existent object should fail objectName3 := randString(60, rand.NewSource(time.Now().UnixNano()), "test-multipart-") args["objectName"] = objectName3 data3 := bytes.Repeat([]byte("a"), 5*1024*1024+1) reader3 := bytes.NewReader(data3) opts3 := minio.PutObjectOptions{} opts3.SetMatchETag("non-existent-etag") _, err = c.PutObject(context.Background(), bucketName, objectName3, reader3, int64(len(data3)), opts3) if err == nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload with SetMatchETag on non-existent object should have failed", nil) return } errResp = minio.ToErrorResponse(err) if errResp.Code != "NoSuchKey" { logError(testName, function, args, startTime, "", fmt.Sprintf("Expected NoSuchKey error (AWS standard for non-existent objects) for multipart, got %s", errResp.Code), err) return } logSuccess(testName, function, args, startTime) } // Test get object seeker from the end, using whence set to '2'. func testGetObjectSeekEnd() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) return } pos, err := r.Seek(-100, 2) if err != nil { logError(testName, function, args, startTime, "", "Object Seek failed", err) return } if pos != st.Size-100 { logError(testName, function, args, startTime, "", "Incorrect position", err) return } buf2 := make([]byte, 100) m, err := readFull(r, buf2) if err != nil { logError(testName, function, args, startTime, "", "Error reading through readFull", err) return } if m != len(buf2) { logError(testName, function, args, startTime, "", "Number of bytes dont match, expected "+string(len(buf2))+" got "+string(m), err) return } hexBuf1 := fmt.Sprintf("%02x", buf[len(buf)-100:]) hexBuf2 := fmt.Sprintf("%02x", buf2[:m]) if hexBuf1 != hexBuf2 { logError(testName, function, args, startTime, "", "Values at same index dont match", err) return } pos, err = r.Seek(-100, 2) if err != nil { logError(testName, function, args, startTime, "", "Object Seek failed", err) return } if pos != st.Size-100 { logError(testName, function, args, startTime, "", "Incorrect position", err) return } if err = r.Close(); err != nil { logError(testName, function, args, startTime, "", "ObjectClose failed", err) return } logSuccess(testName, function, args, startTime) } // Test get object reader to not throw error on being closed twice. func testGetObjectClosedTwice() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) return } if err := crcMatchesName(r, "datafile-33-kB"); err != nil { logError(testName, function, args, startTime, "", "data CRC check failed", err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", "Object Close failed", err) return } if err := r.Close(); err == nil { logError(testName, function, args, startTime, "", "Already closed object. No error returned", err) return } logSuccess(testName, function, args, startTime) } // Test RemoveObjects request where context cancels after timeout func testRemoveObjectsContext() { // Initialize logging params. startTime := time.Now() testName := getFuncName() function := "RemoveObjects(ctx, bucketName, objectsCh)" args := map[string]interface{}{ "bucketName": "", } // Instantiate new minio client. c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate put data. r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) // Multi remove of 20 objects. nrObjects := 20 objectsCh := make(chan minio.ObjectInfo) go func() { defer close(objectsCh) for i := 0; i < nrObjects; i++ { objectName := "sample" + strconv.Itoa(i) + ".txt" info, err := c.PutObject(context.Background(), bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) continue } objectsCh <- minio.ObjectInfo{ Key: info.Key, VersionID: info.VersionID, } } }() // Set context to cancel in 1 nanosecond. ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx defer cancel() // Call RemoveObjects API with short timeout. errorCh := c.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{}) // Check for error. select { case r := <-errorCh: if r.Err == nil { logError(testName, function, args, startTime, "", "RemoveObjects should fail on short timeout", err) return } } // Set context with longer timeout. ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) args["ctx"] = ctx defer cancel() // Perform RemoveObjects with the longer timeout. Expect the removals to succeed. errorCh = c.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{}) select { case r, more := <-errorCh: if more || r.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error", r.Err) return } } logSuccess(testName, function, args, startTime) } // Test removing multiple objects with Remove API func testRemoveMultipleObjects() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "RemoveObjects(bucketName, objectsCh)" args := map[string]interface{}{ "bucketName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) r := bytes.NewReader(bytes.Repeat([]byte("a"), 1)) // Multi remove of 1100 objects nrObjects := 1100 objectsCh := make(chan minio.ObjectInfo) go func() { defer close(objectsCh) // Upload objects and send them to objectsCh for i := 0; i < nrObjects; i++ { objectName := "sample" + strconv.Itoa(i) + ".txt" info, err := c.PutObject(context.Background(), bucketName, objectName, r, 1, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) continue } objectsCh <- minio.ObjectInfo{ Key: info.Key, VersionID: info.VersionID, } } }() // Call RemoveObjects API errorCh := c.RemoveObjects(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{}) // Check if errorCh doesn't receive any error select { case r, more := <-errorCh: if more { logError(testName, function, args, startTime, "", "Unexpected error", r.Err) return } } logSuccess(testName, function, args, startTime) } // Test removing multiple objects with Remove API as iterator func testRemoveMultipleObjectsIter() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "RemoveObjects(bucketName, objectsCh)" args := map[string]interface{}{ "bucketName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) buf := []byte("a") // Multi remove of 1100 objects nrObjects := 1100 objectsIter := func() iter.Seq[minio.ObjectInfo] { return func(yield func(minio.ObjectInfo) bool) { // Upload objects and send them to objectsCh for i := 0; i < nrObjects; i++ { objectName := "sample" + strconv.Itoa(i) + ".txt" info, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), 1, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) continue } if !yield(minio.ObjectInfo{ Key: info.Key, VersionID: info.VersionID, }) { return } } } } // Call RemoveObjects API results, err := c.RemoveObjectsWithIter(context.Background(), bucketName, objectsIter(), minio.RemoveObjectsOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Unexpected error", err) return } for result := range results { if result.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error", result.Err) return } } logSuccess(testName, function, args, startTime) } // Test removing multiple objects and check for results func testRemoveMultipleObjectsWithResult() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "RemoveObjects(bucketName, objectsCh)" args := map[string]interface{}{ "bucketName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupVersionedBucket(bucketName, c) buf := []byte("a") nrObjects := 10 nrLockedObjects := 5 objectsCh := make(chan minio.ObjectInfo) go func() { defer close(objectsCh) // Upload objects and send them to objectsCh for i := 0; i < nrObjects; i++ { objectName := "sample" + strconv.Itoa(i) + ".txt" info, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), 1, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } if i < nrLockedObjects { // t := time.Date(2130, time.April, 25, 14, 0, 0, 0, time.UTC) t := time.Now().Add(5 * time.Minute) m := minio.RetentionMode(minio.Governance) opts := minio.PutObjectRetentionOptions{ GovernanceBypass: false, RetainUntilDate: &t, Mode: &m, VersionID: info.VersionID, } err = c.PutObjectRetention(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "Error setting retention", err) return } } objectsCh <- minio.ObjectInfo{ Key: info.Key, VersionID: info.VersionID, } } }() // Call RemoveObjects API resultCh := c.RemoveObjectsWithResult(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{}) var foundNil, foundErr int for { // Check if errorCh doesn't receive any error select { case deleteRes, ok := <-resultCh: if !ok { goto out } if deleteRes.ObjectName == "" { logError(testName, function, args, startTime, "", "Unexpected object name", nil) return } if deleteRes.ObjectVersionID == "" { logError(testName, function, args, startTime, "", "Unexpected object version ID", nil) return } if deleteRes.Err == nil { foundNil++ } else { foundErr++ } } } out: if foundNil+foundErr != nrObjects { logError(testName, function, args, startTime, "", "Unexpected number of results", nil) return } if foundNil != nrObjects-nrLockedObjects { logError(testName, function, args, startTime, "", "Unexpected number of nil errors", nil) return } if foundErr != nrLockedObjects { logError(testName, function, args, startTime, "", "Unexpected number of errors", nil) return } logSuccess(testName, function, args, startTime) } // Tests FPutObject of a big file to trigger multipart func testFPutObjectMultipart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FPutObject(bucketName, objectName, fileName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "fileName": "", "opts": "", } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. fileName := getMintDataDirFilePath("datafile-129-MB") if fileName == "" { // Make a temp file with minPartSize bytes of data. file, err := os.CreateTemp(os.TempDir(), "FPutObjectTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return } // Upload 2 parts to utilize all 3 'workers' in multipart and still have a part to upload. if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } if err = file.Close(); err != nil { logError(testName, function, args, startTime, "", "File Close failed", err) return } fileName = file.Name() args["fileName"] = fileName } totalSize := dataFileMap["datafile-129-MB"] // Set base object name objectName := bucketName + "FPutObject" + "-standard" args["objectName"] = objectName objectContentType := "testapplication/octet-stream" args["objectContentType"] = objectContentType // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ContentType: objectContentType}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } objInfo, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Unexpected error", err) return } if objInfo.Size != int64(totalSize) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(totalSize))+" got "+string(objInfo.Size), err) return } if objInfo.ContentType != objectContentType && objInfo.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "ContentType doesn't match", err) return } logSuccess(testName, function, args, startTime) } // Tests FPutObject with null contentType (default = application/octet-stream) func testFPutObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FPutObject(bucketName, objectName, fileName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "fileName": "", "opts": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") location := "us-east-1" // Make a new bucket. args["bucketName"] = bucketName args["location"] = location function = "MakeBucket(bucketName, location)" err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: location}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Upload 3 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. // Use different data in part for multipart tests to check parts are uploaded in correct order. fName := getMintDataDirFilePath("datafile-129-MB") if fName == "" { // Make a temp file with minPartSize bytes of data. file, err := os.CreateTemp(os.TempDir(), "FPutObjectTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return } // Upload 3 parts to utilize all 3 'workers' in multipart and still have a part to upload. if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil { logError(testName, function, args, startTime, "", "File copy failed", err) return } // Close the file pro-actively for windows. if err = file.Close(); err != nil { logError(testName, function, args, startTime, "", "File close failed", err) return } defer os.Remove(file.Name()) fName = file.Name() } // Set base object name function = "FPutObject(bucketName, objectName, fileName, opts)" objectName := bucketName + "FPutObject" args["objectName"] = objectName + "-standard" args["fileName"] = fName args["opts"] = minio.PutObjectOptions{ContentType: "application/octet-stream"} // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) ui, err := c.FPutObject(context.Background(), bucketName, objectName+"-standard", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } if ui.Size != int64(dataFileMap["datafile-129-MB"]) { logError(testName, function, args, startTime, "", "FPutObject returned an unexpected upload size", err) return } // Perform FPutObject with no contentType provided (Expecting application/octet-stream) args["objectName"] = objectName + "-Octet" _, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", fName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "File close failed", err) return } srcFile, err := os.Open(fName) if err != nil { logError(testName, function, args, startTime, "", "File open failed", err) return } defer srcFile.Close() // Add extension to temp file name tmpFile, err := os.Create(fName + ".gtar") if err != nil { logError(testName, function, args, startTime, "", "File create failed", err) return } _, err = io.Copy(tmpFile, srcFile) if err != nil { logError(testName, function, args, startTime, "", "File copy failed", err) return } tmpFile.Close() // Perform FPutObject with no contentType provided (Expecting application/x-gtar) args["objectName"] = objectName + "-GTar" args["opts"] = minio.PutObjectOptions{} _, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } // Check headers function = "StatObject(bucketName, objectName, opts)" args["objectName"] = objectName + "-standard" rStandard, err := c.StatObject(context.Background(), bucketName, objectName+"-standard", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if rStandard.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rStandard.ContentType, err) return } function = "StatObject(bucketName, objectName, opts)" args["objectName"] = objectName + "-Octet" rOctet, err := c.StatObject(context.Background(), bucketName, objectName+"-Octet", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if rOctet.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rOctet.ContentType, err) return } function = "StatObject(bucketName, objectName, opts)" args["objectName"] = objectName + "-GTar" rGTar, err := c.StatObject(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" && rGTar.ContentType != "application/x-tar" { logError(testName, function, args, startTime, "", "ContentType does not match, expected application/x-tar or application/octet-stream, got "+rGTar.ContentType, err) return } os.Remove(fName + ".gtar") logSuccess(testName, function, args, startTime) } // Tests FPutObject request when context cancels after timeout func testFPutObjectContext() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FPutObject(bucketName, objectName, fileName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "fileName": "", "opts": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Upload 1 parts worth of data to use multipart upload. // Use different data in part for multipart tests to check parts are uploaded in correct order. fName := getMintDataDirFilePath("datafile-1-MB") if fName == "" { // Make a temp file with 1 MiB bytes of data. file, err := os.CreateTemp(os.TempDir(), "FPutObjectContextTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return } // Upload 1 parts to trigger multipart upload if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil { logError(testName, function, args, startTime, "", "File copy failed", err) return } // Close the file pro-actively for windows. if err = file.Close(); err != nil { logError(testName, function, args, startTime, "", "File close failed", err) return } defer os.Remove(file.Name()) fName = file.Name() } // Set base object name objectName := bucketName + "FPutObjectContext" args["objectName"] = objectName ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx defer cancel() // Perform FPutObject with contentType provided (Expecting application/octet-stream) _, err = c.FPutObject(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err == nil { logError(testName, function, args, startTime, "", "FPutObject should fail on short timeout", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() // Perform FPutObject with a long timeout. Expect the put object to succeed _, err = c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject shouldn't fail on long timeout", err) return } _, err = c.StatObject(context.Background(), bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } logSuccess(testName, function, args, startTime) } // Tests FPutObject request when context cancels after timeout func testFPutObjectContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FPutObjectContext(ctx, bucketName, objectName, fileName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{ContentType:objectContentType}", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Upload 1 parts worth of data to use multipart upload. // Use different data in part for multipart tests to check parts are uploaded in correct order. fName := getMintDataDirFilePath("datafile-1-MB") if fName == "" { // Make a temp file with 1 MiB bytes of data. file, err := os.CreateTemp(os.TempDir(), "FPutObjectContextTest") if err != nil { logError(testName, function, args, startTime, "", "Temp file creation failed", err) return } // Upload 1 parts to trigger multipart upload if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil { logError(testName, function, args, startTime, "", "File copy failed", err) return } // Close the file pro-actively for windows. if err = file.Close(); err != nil { logError(testName, function, args, startTime, "", "File close failed", err) return } defer os.Remove(file.Name()) fName = file.Name() } // Set base object name objectName := bucketName + "FPutObjectContext" args["objectName"] = objectName ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx defer cancel() // Perform FPutObject with contentType provided (Expecting application/octet-stream) _, err = c.FPutObject(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err == nil { logError(testName, function, args, startTime, "", "FPutObject should fail on short timeout", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() // Perform FPutObject with a long timeout. Expect the put object to succeed _, err = c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject shouldn't fail on longer timeout", err) return } _, err = c.StatObject(context.Background(), bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } logSuccess(testName, function, args, startTime) } // Test validates putObject with context to see if request cancellation is honored. func testPutObjectContext() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(ctx, bucketName, objectName, fileName, opts)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", "opts": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket call failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) args["objectName"] = objectName ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) cancel() args["ctx"] = ctx args["opts"] = minio.PutObjectOptions{ContentType: "binary/octet-stream"} _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err == nil { logError(testName, function, args, startTime, "", "PutObject should fail on short timeout", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) args["ctx"] = ctx defer cancel() reader = getDataReader("datafile-33-kB") defer reader.Close() _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject with long timeout failed", err) return } logSuccess(testName, function, args, startTime) } // Tests get object with s3zip extensions. func testGetObjectS3Zip() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{"x-minio-extract": true} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer func() { // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } }() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + ".zip" args["objectName"] = objectName var zipFile bytes.Buffer zw := zip.NewWriter(&zipFile) rng := rand.New(rand.NewSource(0xc0cac01a)) const nFiles = 500 for i := 0; i <= nFiles; i++ { if i == nFiles { // Make one large, compressible file. i = 1000000 } b := make([]byte, i) if i < nFiles { rng.Read(b) } wc, err := zw.Create(fmt.Sprintf("test/small/file-%d.bin", i)) if err != nil { logError(testName, function, args, startTime, "", "zw.Create failed", err) return } wc.Write(b) } err = zw.Close() if err != nil { logError(testName, function, args, startTime, "", "zw.Close failed", err) return } buf := zipFile.Bytes() // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat object failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(len(buf))+", got "+string(st.Size), err) return } r.Close() zr, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf))) if err != nil { logError(testName, function, args, startTime, "", "zip.NewReader failed", err) return } lOpts := minio.ListObjectsOptions{} lOpts.Set("x-minio-extract", "true") lOpts.Prefix = objectName + "/" lOpts.Recursive = true list := c.ListObjects(context.Background(), bucketName, lOpts) listed := map[string]minio.ObjectInfo{} for item := range list { if item.Err != nil { break } listed[item.Key] = item } if len(listed) == 0 { // Assume we are running against non-minio. args["SKIPPED"] = true logIgnored(testName, function, args, startTime, "s3zip does not appear to be present") return } for _, file := range zr.File { if file.FileInfo().IsDir() { continue } args["zipfile"] = file.Name zfr, err := file.Open() if err != nil { logError(testName, function, args, startTime, "", "file.Open failed", err) return } want, err := io.ReadAll(zfr) if err != nil { logError(testName, function, args, startTime, "", "fzip file read failed", err) return } opts := minio.GetObjectOptions{} opts.Set("x-minio-extract", "true") key := path.Join(objectName, file.Name) r, err = c.GetObject(context.Background(), bucketName, key, opts) if err != nil { terr := minio.ToErrorResponse(err) if terr.StatusCode != http.StatusNotFound { logError(testName, function, args, startTime, "", "GetObject failed", err) } return } got, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } r.Close() if !bytes.Equal(want, got) { logError(testName, function, args, startTime, "", "Content mismatch", err) return } oi, ok := listed[key] if !ok { logError(testName, function, args, startTime, "", "Object Missing", fmt.Errorf("%s not present in listing", key)) return } if int(oi.Size) != len(got) { logError(testName, function, args, startTime, "", "Object Size Incorrect", fmt.Errorf("listing %d, read %d", oi.Size, len(got))) return } delete(listed, key) } delete(args, "zipfile") if len(listed) > 0 { logError(testName, function, args, startTime, "", "Extra listed objects", fmt.Errorf("left over: %v", listed)) return } logSuccess(testName, function, args, startTime) } // Tests get object ReaderSeeker interface methods. func testGetObjectReadSeekFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer func() { // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } }() // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat object failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } // This following function helps us to compare data from the reader after seek // with the data from the original buffer cmpData := func(r io.Reader, start, end int) { if end-start == 0 { return } buffer := bytes.NewBuffer([]byte{}) if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "CopyN failed", err) return } } if !bytes.Equal(buf[start:end], buffer.Bytes()) { logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) return } } // Generic seek error for errors other than io.EOF seekErr := errors.New("seek error") testCases := []struct { offset int64 whence int pos int64 err error shouldCmp bool start int end int }{ // Start from offset 0, fetch data and compare {0, 0, 0, nil, true, 0, 0}, // Start from offset 2048, fetch data and compare {2048, 0, 2048, nil, true, 2048, bufSize}, // Start from offset larger than possible {int64(bufSize) + 1024, 0, 0, seekErr, false, 0, 0}, // Move to offset 0 without comparing {0, 0, 0, nil, false, 0, 0}, // Move one step forward and compare {1, 1, 1, nil, true, 1, bufSize}, // Move larger than possible {int64(bufSize), 1, 0, seekErr, false, 0, 0}, // Provide negative offset with CUR_SEEK {int64(-1), 1, 0, seekErr, false, 0, 0}, // Test with whence SEEK_END and with positive offset {1024, 2, int64(bufSize) - 1024, io.EOF, true, 0, 0}, // Test with whence SEEK_END and with negative offset {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, // Test with whence SEEK_END and with large negative offset {-int64(bufSize) * 2, 2, 0, seekErr, true, 0, 0}, } for i, testCase := range testCases { // Perform seek operation n, err := r.Seek(testCase.offset, testCase.whence) // We expect an error if testCase.err == seekErr && err == nil { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err) return } // We expect a specific error if testCase.err != seekErr && testCase.err != err { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err) return } // If we expect an error go to the next loop if testCase.err != nil { continue } // Check the returned seek pos if n != testCase.pos { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", number of bytes seeked does not match, expected "+string(testCase.pos)+", got "+string(n), err) return } // Compare only if shouldCmp is activated if testCase.shouldCmp { cmpData(r, testCase.start, testCase.end) } } logSuccess(testName, function, args, startTime) } // Tests get object ReaderAt interface methods. func testGetObjectReadAtFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } offset := int64(2048) // read directly buf1 := make([]byte, 512) buf2 := make([]byte, 512) buf3 := make([]byte, 512) buf4 := make([]byte, 512) // Test readAt before stat is called such that objectInfo doesn't change. m, err := r.ReadAt(buf1, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf1) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) return } if !bytes.Equal(buf1, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } m, err = r.ReadAt(buf2, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf2) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) return } if !bytes.Equal(buf2, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf3, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf3) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) return } if !bytes.Equal(buf3, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf4, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf4) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) return } if !bytes.Equal(buf4, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } buf5 := make([]byte, len(buf)) // Read the whole object. m, err = r.ReadAt(buf5, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } if m != len(buf5) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) return } if !bytes.Equal(buf, buf5) { logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) return } buf6 := make([]byte, len(buf)+1) // Read the whole object and beyond. _, err = r.ReadAt(buf6, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } logSuccess(testName, function, args, startTime) } // Reproduces issue https://github.com/minio/minio-go/issues/1137 func testGetObjectReadAtWhenEOFWasReached() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // read directly buf1 := make([]byte, len(buf)) buf2 := make([]byte, 512) m, err := r.Read(buf1) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "Read failed", err) return } } if m != len(buf1) { logError(testName, function, args, startTime, "", "Read read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) return } if !bytes.Equal(buf1, buf) { logError(testName, function, args, startTime, "", "Incorrect count of Read data", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } m, err = r.ReadAt(buf2, 512) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf2) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) return } if !bytes.Equal(buf2, buf[512:1024]) { logError(testName, function, args, startTime, "", "Incorrect count of ReadAt data", err) return } logSuccess(testName, function, args, startTime) } // Test Presigned Post Policy func testPresignedPostPolicy() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PresignedPostPolicy(policy)" args := map[string]interface{}{ "policy": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") // Azure requires the key to not start with a number metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user") metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } policy := minio.NewPostPolicy() policy.SetBucket(bucketName) policy.SetKey(objectName) policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days policy.SetContentType("binary/octet-stream") policy.SetContentLengthRange(10, 1024*1024) policy.SetUserMetadata(metadataKey, metadataValue) policy.SetContentEncoding("gzip") // Add CRC32C checksum := minio.ChecksumCRC32C.ChecksumBytes(buf) err = policy.SetChecksum(checksum) if err != nil { logError(testName, function, args, startTime, "", "SetChecksum failed", err) return } args["policy"] = policy.String() presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(context.Background(), policy) if err != nil { logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err) return } var formBuf bytes.Buffer writer := multipart.NewWriter(&formBuf) for k, v := range formData { writer.WriteField(k, v) } // Get a 33KB file to upload and test if set post policy works filePath := getMintDataDirFilePath("datafile-33-kB") if filePath == "" { // Make a temp file with 33 KB data. file, err := os.CreateTemp(os.TempDir(), "PresignedPostPolicyTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return } if _, err = io.Copy(file, getDataReader("datafile-33-kB")); err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } if err = file.Close(); err != nil { logError(testName, function, args, startTime, "", "File Close failed", err) return } filePath = file.Name() } // add file to post request f, err := os.Open(filePath) defer f.Close() if err != nil { logError(testName, function, args, startTime, "", "File open failed", err) return } w, err := writer.CreateFormFile("file", filePath) if err != nil { logError(testName, function, args, startTime, "", "CreateFormFile failed", err) return } _, err = io.Copy(w, f) if err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } writer.Close() httpClient := &http.Client{ // Setting a sensible time out of 30secs to wait for response // headers. Request is pro-actively canceled after 30secs // with no response. Timeout: 30 * time.Second, Transport: createHTTPTransport(), } args["url"] = presignedPostPolicyURL.String() req, err := http.NewRequest(http.MethodPost, presignedPostPolicyURL.String(), bytes.NewReader(formBuf.Bytes())) if err != nil { logError(testName, function, args, startTime, "", "Http request failed", err) return } req.Header.Set("Content-Type", writer.FormDataContentType()) // make post request with correct form data res, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "Http request failed", err) return } defer res.Body.Close() if res.StatusCode != http.StatusNoContent { logError(testName, function, args, startTime, "", "Http request failed", errors.New(res.Status)) return } // expected path should be absolute path of the object var scheme string if mustParseBool(os.Getenv(enableHTTPS)) { scheme = "https://" } else { scheme = "http://" } expectedLocation := scheme + os.Getenv(serverEndpoint) + "/" + bucketName + "/" + objectName expectedLocationBucketDNS := scheme + bucketName + "." + os.Getenv(serverEndpoint) + "/" + objectName if !strings.Contains(expectedLocation, ".amazonaws.com/") { // Test when not against AWS S3. if val, ok := res.Header["Location"]; ok { if val[0] != expectedLocation && val[0] != expectedLocationBucketDNS { logError(testName, function, args, startTime, "", fmt.Sprintf("Location in header response is incorrect. Want %q or %q, got %q", expectedLocation, expectedLocationBucketDNS, val[0]), err) return } } else { logError(testName, function, args, startTime, "", "Location not found in header response", err) return } } wantChecksumCrc32c := checksum.Encoded() if got := res.Header.Get("X-Amz-Checksum-Crc32c"); got != wantChecksumCrc32c { logError(testName, function, args, startTime, "", fmt.Sprintf("Want checksum %q, got %q", wantChecksumCrc32c, got), nil) return } // Ensure that when we subsequently GetObject, the checksum is returned gopts := minio.GetObjectOptions{Checksum: true} r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.ChecksumCRC32C != wantChecksumCrc32c { logError(testName, function, args, startTime, "", fmt.Sprintf("Want checksum %s, got %s", wantChecksumCrc32c, st.ChecksumCRC32C), nil) return } logSuccess(testName, function, args, startTime) } // testPresignedPostPolicyWrongFile tests that when we have a policy with a checksum, we cannot POST the wrong file func testPresignedPostPolicyWrongFile() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PresignedPostPolicy(policy)" args := map[string]interface{}{ "policy": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") // Azure requires the key to not start with a number metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user") metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") policy := minio.NewPostPolicy() policy.SetBucket(bucketName) policy.SetKey(objectName) policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days policy.SetContentType("binary/octet-stream") policy.SetContentLengthRange(10, 1024*1024) policy.SetUserMetadata(metadataKey, metadataValue) // Add CRC32C of some data that the policy will explicitly allow. checksum := minio.ChecksumCRC32C.ChecksumBytes([]byte{0x01, 0x02, 0x03}) err = policy.SetChecksum(checksum) if err != nil { logError(testName, function, args, startTime, "", "SetChecksum failed", err) return } args["policy"] = policy.String() presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(context.Background(), policy) if err != nil { logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err) return } // At this stage, we have a policy that allows us to upload for a specific checksum. // Test that uploading datafile-10-kB, with a different checksum, fails as expected filePath := getMintDataDirFilePath("datafile-10-kB") if filePath == "" { // Make a temp file with 10 KB data. file, err := os.CreateTemp(os.TempDir(), "PresignedPostPolicyTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return } if _, err = io.Copy(file, getDataReader("datafile-10-kB")); err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } if err = file.Close(); err != nil { logError(testName, function, args, startTime, "", "File Close failed", err) return } filePath = file.Name() } fileReader := getDataReader("datafile-10-kB") defer fileReader.Close() buf10k, err := io.ReadAll(fileReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } otherChecksum := minio.ChecksumCRC32C.ChecksumBytes(buf10k) var formBuf bytes.Buffer writer := multipart.NewWriter(&formBuf) for k, v := range formData { if k == "x-amz-checksum-crc32c" { v = otherChecksum.Encoded() } writer.WriteField(k, v) } // Add file to post request f, err := os.Open(filePath) defer f.Close() if err != nil { logError(testName, function, args, startTime, "", "File open failed", err) return } w, err := writer.CreateFormFile("file", filePath) if err != nil { logError(testName, function, args, startTime, "", "CreateFormFile failed", err) return } _, err = io.Copy(w, f) if err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } writer.Close() httpClient := &http.Client{ Timeout: 30 * time.Second, Transport: createHTTPTransport(), } args["url"] = presignedPostPolicyURL.String() req, err := http.NewRequest(http.MethodPost, presignedPostPolicyURL.String(), bytes.NewReader(formBuf.Bytes())) if err != nil { logError(testName, function, args, startTime, "", "HTTP request failed", err) return } req.Header.Set("Content-Type", writer.FormDataContentType()) // Make the POST request with the form data. res, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "HTTP request failed", err) return } defer res.Body.Close() if res.StatusCode != http.StatusForbidden { logError(testName, function, args, startTime, "", "HTTP request unexpected status", errors.New(res.Status)) return } // Read the response body, ensure it has checksum failure message resBody, err := io.ReadAll(res.Body) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Normalize the response body, because S3 uses quotes around the policy condition components // in the error message, MinIO does not. resBodyStr := strings.ReplaceAll(string(resBody), `"`, "") if !strings.Contains(resBodyStr, "Policy Condition failed: [eq, $x-amz-checksum-crc32c, 8TDyHg=") { logError(testName, function, args, startTime, "", "Unexpected response body", errors.New(resBodyStr)) return } logSuccess(testName, function, args, startTime) } // testPresignedPostPolicyEmptyFileName tests that an empty file name in the presigned post policy func testPresignedPostPolicyEmptyFileName() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PresignedPostPolicy(policy)" args := map[string]interface{}{ "policy": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") // Azure requires the key to not start with a number metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user") metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } policy := minio.NewPostPolicy() policy.SetBucket(bucketName) policy.SetKey(objectName) policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days policy.SetContentType("binary/octet-stream") policy.SetContentLengthRange(10, 1024*1024) policy.SetUserMetadata(metadataKey, metadataValue) policy.SetContentEncoding("gzip") // Add CRC32C checksum := minio.ChecksumCRC32C.ChecksumBytes(buf) err = policy.SetChecksum(checksum) if err != nil { logError(testName, function, args, startTime, "", "SetChecksum failed", err) return } args["policy"] = policy.String() presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(context.Background(), policy) if err != nil { logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err) return } var formBuf bytes.Buffer writer := multipart.NewWriter(&formBuf) for k, v := range formData { writer.WriteField(k, v) } // Get a 33KB file to upload and test if set post policy works filePath := getMintDataDirFilePath("datafile-33-kB") if filePath == "" { // Make a temp file with 33 KB data. file, err := os.CreateTemp(os.TempDir(), "PresignedPostPolicyTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return } if _, err = io.Copy(file, getDataReader("datafile-33-kB")); err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } if err = file.Close(); err != nil { logError(testName, function, args, startTime, "", "File Close failed", err) return } filePath = file.Name() } // add file to post request f, err := os.Open(filePath) defer f.Close() if err != nil { logError(testName, function, args, startTime, "", "File open failed", err) return } w, err := writer.CreateFormFile("", filePath) if err != nil { logError(testName, function, args, startTime, "", "CreateFormFile failed", err) return } _, err = io.Copy(w, f) if err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } writer.Close() httpClient := &http.Client{ // Setting a sensible time out of 30secs to wait for response // headers. Request is pro-actively canceled after 30secs // with no response. Timeout: 30 * time.Second, Transport: createHTTPTransport(), } args["url"] = presignedPostPolicyURL.String() req, err := http.NewRequest(http.MethodPost, presignedPostPolicyURL.String(), bytes.NewReader(formBuf.Bytes())) if err != nil { logError(testName, function, args, startTime, "", "Http request failed", err) return } req.Header.Set("Content-Type", writer.FormDataContentType()) // make post request with correct form data res, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "Http request failed", err) return } defer res.Body.Close() if res.StatusCode != http.StatusBadRequest { logError(testName, function, args, startTime, "", "Http request failed", errors.New(res.Status)) return } body, err := io.ReadAll(res.Body) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } if !strings.Contains(string(body), "MalformedPOSTRequest") { logError(testName, function, args, startTime, "", "Invalid error from server", errors.New(string(body))) } logSuccess(testName, function, args, startTime) } // Tests copy object func testCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(dst, src)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Make a new bucket in 'us-east-1' (destination bucket). err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName+"-copy", c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } // Check the various fields of source object against destination object. objInfo, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } // Copy Source src := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, // Set copy conditions. MatchETag: objInfo.ETag, MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), } args["src"] = src dst := minio.CopyDestOptions{ Bucket: bucketName + "-copy", Object: objectName + "-copy", } // Perform the Copy if _, err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Source object r, err = c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } // Destination object readerCopy, err := c.GetObject(context.Background(), bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } // Check the various fields of source object against destination object. objInfo, err = r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } objInfoCopy, err := readerCopy.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if objInfo.Size != objInfoCopy.Size { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+", got "+string(objInfo.Size), err) return } if err := crcMatchesName(r, "datafile-33-kB"); err != nil { logError(testName, function, args, startTime, "", "data CRC check failed", err) return } if err := crcMatchesName(readerCopy, "datafile-33-kB"); err != nil { logError(testName, function, args, startTime, "", "copy data CRC check failed", err) return } // Close all the get readers before proceeding with CopyObject operations. r.Close() readerCopy.Close() // CopyObject again but with wrong conditions src = minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, MatchUnmodifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), NoMatchETag: objInfo.ETag, } // Perform the Copy which should fail _, err = c.CopyObject(context.Background(), dst, src) if err == nil { logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) return } src = minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, } dst = minio.CopyDestOptions{ Bucket: bucketName, Object: objectName, ReplaceMetadata: true, UserMetadata: map[string]string{ "Copy": "should be same", }, } args["dst"] = dst args["src"] = src _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject shouldn't fail", err) return } oi, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } stOpts := minio.StatObjectOptions{} stOpts.SetMatchETag(oi.ETag) objInfo, err = c.StatObject(context.Background(), bucketName, objectName, stOpts) if err != nil { logError(testName, function, args, startTime, "", "CopyObject ETag should match and not fail", err) return } if objInfo.Metadata.Get("x-amz-meta-copy") != "should be same" { logError(testName, function, args, startTime, "", "CopyObject modified metadata should match", err) return } logSuccess(testName, function, args, startTime) } // Tests SSE-C get object ReaderSeeker interface methods. func testSSECEncryptedGetObjectReadSeekFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer func() { // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } }() // Generate 129MiB of data. bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), }) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer r.Close() st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat object failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } // This following function helps us to compare data from the reader after seek // with the data from the original buffer cmpData := func(r io.Reader, start, end int) { if end-start == 0 { return } buffer := bytes.NewBuffer([]byte{}) if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "CopyN failed", err) return } } if !bytes.Equal(buf[start:end], buffer.Bytes()) { logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) return } } testCases := []struct { offset int64 whence int pos int64 err error shouldCmp bool start int end int }{ // Start from offset 0, fetch data and compare {0, 0, 0, nil, true, 0, 0}, // Start from offset 2048, fetch data and compare {2048, 0, 2048, nil, true, 2048, bufSize}, // Start from offset larger than possible {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, // Move to offset 0 without comparing {0, 0, 0, nil, false, 0, 0}, // Move one step forward and compare {1, 1, 1, nil, true, 1, bufSize}, // Move larger than possible {int64(bufSize), 1, 0, io.EOF, false, 0, 0}, // Provide negative offset with CUR_SEEK {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0}, // Test with whence SEEK_END and with positive offset {1024, 2, 0, io.EOF, false, 0, 0}, // Test with whence SEEK_END and with negative offset {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, // Test with whence SEEK_END and with large negative offset {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0}, // Test with invalid whence {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0}, } for i, testCase := range testCases { // Perform seek operation n, err := r.Seek(testCase.offset, testCase.whence) if err != nil && testCase.err == nil { // We expected success. logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } if err == nil && testCase.err != nil { // We expected failure, but got success. logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } if err != nil && testCase.err != nil { if err.Error() != testCase.err.Error() { // We expect a specific error logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } } // Check the returned seek pos if n != testCase.pos { logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err) return } // Compare only if shouldCmp is activated if testCase.shouldCmp { cmpData(r, testCase.start, testCase.end) } } logSuccess(testName, function, args, startTime) } // Tests SSE-S3 get object ReaderSeeker interface methods. func testSSES3EncryptedGetObjectReadSeekFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer func() { // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } }() // Generate 129MiB of data. bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.NewSSE(), }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer r.Close() st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat object failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } // This following function helps us to compare data from the reader after seek // with the data from the original buffer cmpData := func(r io.Reader, start, end int) { if end-start == 0 { return } buffer := bytes.NewBuffer([]byte{}) if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "CopyN failed", err) return } } if !bytes.Equal(buf[start:end], buffer.Bytes()) { logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) return } } testCases := []struct { offset int64 whence int pos int64 err error shouldCmp bool start int end int }{ // Start from offset 0, fetch data and compare {0, 0, 0, nil, true, 0, 0}, // Start from offset 2048, fetch data and compare {2048, 0, 2048, nil, true, 2048, bufSize}, // Start from offset larger than possible {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, // Move to offset 0 without comparing {0, 0, 0, nil, false, 0, 0}, // Move one step forward and compare {1, 1, 1, nil, true, 1, bufSize}, // Move larger than possible {int64(bufSize), 1, 0, io.EOF, false, 0, 0}, // Provide negative offset with CUR_SEEK {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0}, // Test with whence SEEK_END and with positive offset {1024, 2, 0, io.EOF, false, 0, 0}, // Test with whence SEEK_END and with negative offset {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, // Test with whence SEEK_END and with large negative offset {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0}, // Test with invalid whence {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0}, } for i, testCase := range testCases { // Perform seek operation n, err := r.Seek(testCase.offset, testCase.whence) if err != nil && testCase.err == nil { // We expected success. logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } if err == nil && testCase.err != nil { // We expected failure, but got success. logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } if err != nil && testCase.err != nil { if err.Error() != testCase.err.Error() { // We expect a specific error logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } } // Check the returned seek pos if n != testCase.pos { logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err) return } // Compare only if shouldCmp is activated if testCase.shouldCmp { cmpData(r, testCase.start, testCase.end) } } logSuccess(testName, function, args, startTime) } // Tests SSE-C get object ReaderAt interface methods. func testSSECEncryptedGetObjectReadAtFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 129MiB of data. bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } defer r.Close() offset := int64(2048) // read directly buf1 := make([]byte, 512) buf2 := make([]byte, 512) buf3 := make([]byte, 512) buf4 := make([]byte, 512) // Test readAt before stat is called such that objectInfo doesn't change. m, err := r.ReadAt(buf1, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf1) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) return } if !bytes.Equal(buf1, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } m, err = r.ReadAt(buf2, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf2) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) return } if !bytes.Equal(buf2, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf3, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf3) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) return } if !bytes.Equal(buf3, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf4, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf4) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) return } if !bytes.Equal(buf4, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } buf5 := make([]byte, len(buf)) // Read the whole object. m, err = r.ReadAt(buf5, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } if m != len(buf5) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) return } if !bytes.Equal(buf, buf5) { logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) return } buf6 := make([]byte, len(buf)+1) // Read the whole object and beyond. _, err = r.ReadAt(buf6, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } logSuccess(testName, function, args, startTime) } // Tests SSE-S3 get object ReaderAt interface methods. func testSSES3EncryptedGetObjectReadAtFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 129MiB of data. bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.NewSSE(), }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } defer r.Close() offset := int64(2048) // read directly buf1 := make([]byte, 512) buf2 := make([]byte, 512) buf3 := make([]byte, 512) buf4 := make([]byte, 512) // Test readAt before stat is called such that objectInfo doesn't change. m, err := r.ReadAt(buf1, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf1) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) return } if !bytes.Equal(buf1, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } m, err = r.ReadAt(buf2, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf2) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) return } if !bytes.Equal(buf2, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf3, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf3) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) return } if !bytes.Equal(buf3, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf4, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf4) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) return } if !bytes.Equal(buf4, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } buf5 := make([]byte, len(buf)) // Read the whole object. m, err = r.ReadAt(buf5, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } if m != len(buf5) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) return } if !bytes.Equal(buf, buf5) { logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) return } buf6 := make([]byte, len(buf)+1) // Read the whole object and beyond. _, err = r.ReadAt(buf6, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } logSuccess(testName, function, args, startTime) } // testSSECEncryptionPutGet tests encryption with customer provided encryption keys func testSSECEncryptionPutGet() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutEncryptedObject(bucketName, objectName, reader, sse)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "sse": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) testCases := []struct { buf []byte }{ {buf: bytes.Repeat([]byte("F"), 1)}, {buf: bytes.Repeat([]byte("F"), 15)}, {buf: bytes.Repeat([]byte("F"), 16)}, {buf: bytes.Repeat([]byte("F"), 17)}, {buf: bytes.Repeat([]byte("F"), 31)}, {buf: bytes.Repeat([]byte("F"), 32)}, {buf: bytes.Repeat([]byte("F"), 33)}, {buf: bytes.Repeat([]byte("F"), 1024)}, {buf: bytes.Repeat([]byte("F"), 1024*2)}, {buf: bytes.Repeat([]byte("F"), 1024*1024)}, } const password = "correct horse battery staple" // https://xkcd.com/936/ for i, testCase := range testCases { // Generate a random object name objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Secured object sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) args["sse"] = sse // Put encrypted data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) return } defer r.Close() // Compare the sent object with the received one recvBuffer := bytes.NewBuffer([]byte{}) if _, err = io.Copy(recvBuffer, r); err != nil { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) return } if recvBuffer.Len() != len(testCase.buf) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) return } if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) return } logSuccess(testName, function, args, startTime) } logSuccess(testName, function, args, startTime) } // TestEncryptionFPut tests encryption with customer specified encryption keys func testSSECEncryptionFPut() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "filePath": "", "contentType": "", "sse": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Object custom metadata customContentType := "custom/contenttype" args["metadata"] = customContentType testCases := []struct { buf []byte }{ {buf: bytes.Repeat([]byte("F"), 0)}, {buf: bytes.Repeat([]byte("F"), 1)}, {buf: bytes.Repeat([]byte("F"), 15)}, {buf: bytes.Repeat([]byte("F"), 16)}, {buf: bytes.Repeat([]byte("F"), 17)}, {buf: bytes.Repeat([]byte("F"), 31)}, {buf: bytes.Repeat([]byte("F"), 32)}, {buf: bytes.Repeat([]byte("F"), 33)}, {buf: bytes.Repeat([]byte("F"), 1024)}, {buf: bytes.Repeat([]byte("F"), 1024*2)}, {buf: bytes.Repeat([]byte("F"), 1024*1024)}, } const password = "correct horse battery staple" // https://xkcd.com/936/ for i, testCase := range testCases { // Generate a random object name objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Secured object sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) args["sse"] = sse // Generate a random file name. fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") file, err := os.Create(fileName) if err != nil { logError(testName, function, args, startTime, "", "file create failed", err) return } _, err = file.Write(testCase.buf) if err != nil { logError(testName, function, args, startTime, "", "file write failed", err) return } file.Close() // Put encrypted data if _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) return } defer r.Close() // Compare the sent object with the received one recvBuffer := bytes.NewBuffer([]byte{}) if _, err = io.Copy(recvBuffer, r); err != nil { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) return } if recvBuffer.Len() != len(testCase.buf) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) return } if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) return } os.Remove(fileName) } logSuccess(testName, function, args, startTime) } // testSSES3EncryptionPutGet tests SSE-S3 encryption func testSSES3EncryptionPutGet() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutEncryptedObject(bucketName, objectName, reader, sse)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "sse": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) testCases := []struct { buf []byte }{ {buf: bytes.Repeat([]byte("F"), 1)}, {buf: bytes.Repeat([]byte("F"), 15)}, {buf: bytes.Repeat([]byte("F"), 16)}, {buf: bytes.Repeat([]byte("F"), 17)}, {buf: bytes.Repeat([]byte("F"), 31)}, {buf: bytes.Repeat([]byte("F"), 32)}, {buf: bytes.Repeat([]byte("F"), 33)}, {buf: bytes.Repeat([]byte("F"), 1024)}, {buf: bytes.Repeat([]byte("F"), 1024*2)}, {buf: bytes.Repeat([]byte("F"), 1024*1024)}, } for i, testCase := range testCases { // Generate a random object name objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Secured object sse := encrypt.NewSSE() args["sse"] = sse // Put encrypted data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) return } // Read the data back without any encryption headers r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) return } defer r.Close() // Compare the sent object with the received one recvBuffer := bytes.NewBuffer([]byte{}) if _, err = io.Copy(recvBuffer, r); err != nil { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) return } if recvBuffer.Len() != len(testCase.buf) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) return } if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) return } logSuccess(testName, function, args, startTime) } logSuccess(testName, function, args, startTime) } // TestSSES3EncryptionFPut tests server side encryption func testSSES3EncryptionFPut() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "filePath": "", "contentType": "", "sse": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Object custom metadata customContentType := "custom/contenttype" args["metadata"] = customContentType testCases := []struct { buf []byte }{ {buf: bytes.Repeat([]byte("F"), 0)}, {buf: bytes.Repeat([]byte("F"), 1)}, {buf: bytes.Repeat([]byte("F"), 15)}, {buf: bytes.Repeat([]byte("F"), 16)}, {buf: bytes.Repeat([]byte("F"), 17)}, {buf: bytes.Repeat([]byte("F"), 31)}, {buf: bytes.Repeat([]byte("F"), 32)}, {buf: bytes.Repeat([]byte("F"), 33)}, {buf: bytes.Repeat([]byte("F"), 1024)}, {buf: bytes.Repeat([]byte("F"), 1024*2)}, {buf: bytes.Repeat([]byte("F"), 1024*1024)}, } for i, testCase := range testCases { // Generate a random object name objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Secured object sse := encrypt.NewSSE() args["sse"] = sse // Generate a random file name. fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") file, err := os.Create(fileName) if err != nil { logError(testName, function, args, startTime, "", "file create failed", err) return } _, err = file.Write(testCase.buf) if err != nil { logError(testName, function, args, startTime, "", "file write failed", err) return } file.Close() // Put encrypted data if _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) return } defer r.Close() // Compare the sent object with the received one recvBuffer := bytes.NewBuffer([]byte{}) if _, err = io.Copy(recvBuffer, r); err != nil { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) return } if recvBuffer.Len() != len(testCase.buf) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) return } if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) return } os.Remove(fileName) } logSuccess(testName, function, args, startTime) } func testBucketNotification() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "SetBucketNotification(bucketName)" args := map[string]interface{}{ "bucketName": "", } if os.Getenv("NOTIFY_BUCKET") == "" || os.Getenv("NOTIFY_SERVICE") == "" || os.Getenv("NOTIFY_REGION") == "" || os.Getenv("NOTIFY_ACCOUNTID") == "" || os.Getenv("NOTIFY_RESOURCE") == "" { logIgnored(testName, function, args, startTime, "Skipped notification test as it is not configured") return } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } bucketName := os.Getenv("NOTIFY_BUCKET") args["bucketName"] = bucketName topicArn := notification.NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE")) queueArn := notification.NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource") topicConfig := notification.NewConfig(topicArn) topicConfig.AddEvents(notification.ObjectCreatedAll, notification.ObjectRemovedAll) topicConfig.AddFilterSuffix("jpg") queueConfig := notification.NewConfig(queueArn) queueConfig.AddEvents(notification.ObjectCreatedAll) queueConfig.AddFilterPrefix("photos/") config := notification.Configuration{} config.AddTopic(topicConfig) // Add the same topicConfig again, should have no effect // because it is duplicated config.AddTopic(topicConfig) if len(config.TopicConfigs) != 1 { logError(testName, function, args, startTime, "", "Duplicate entry added", err) return } // Add and remove a queue config config.AddQueue(queueConfig) config.RemoveQueueByArn(queueArn) err = c.SetBucketNotification(context.Background(), bucketName, config) if err != nil { logError(testName, function, args, startTime, "", "SetBucketNotification failed", err) return } config, err = c.GetBucketNotification(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketNotification failed", err) return } if len(config.TopicConfigs) != 1 { logError(testName, function, args, startTime, "", "Topic config is empty", err) return } if config.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" { logError(testName, function, args, startTime, "", "Couldn't get the suffix", err) return } err = c.RemoveAllBucketNotification(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "RemoveAllBucketNotification failed", err) return } // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Tests comprehensive list of all methods. func testFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "testFunctional()" functionAll := "" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, nil, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket. function = "MakeBucket(bucketName, region)" functionAll = "MakeBucket(bucketName, region)" args["bucketName"] = bucketName err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) defer cleanupBucket(bucketName, c) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } // Generate a random file name. fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") file, err := os.Create(fileName) if err != nil { logError(testName, function, args, startTime, "", "File creation failed", err) return } for i := 0; i < 3; i++ { buf := make([]byte, rand.Intn(1<<19)) _, err = file.Write(buf) if err != nil { logError(testName, function, args, startTime, "", "File write failed", err) return } } file.Close() // Verify if bucket exits and you have access. var exists bool function = "BucketExists(bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } exists, err = c.BucketExists(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "BucketExists failed", err) return } if !exists { logError(testName, function, args, startTime, "", "Could not find the bucket", err) return } // Asserting the default bucket policy. function = "GetBucketPolicy(ctx, bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } nilPolicy, err := c.GetBucketPolicy(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return } if nilPolicy != "" { logError(testName, function, args, startTime, "", "policy should be set to nil", err) return } // Set the bucket policy to 'public readonly'. function = "SetBucketPolicy(bucketName, readOnlyPolicy)" functionAll += ", " + function readOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` args = map[string]interface{}{ "bucketName": bucketName, "bucketPolicy": readOnlyPolicy, } err = c.SetBucketPolicy(context.Background(), bucketName, readOnlyPolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // should return policy `readonly`. function = "GetBucketPolicy(ctx, bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } _, err = c.GetBucketPolicy(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return } // Make the bucket 'public writeonly'. function = "SetBucketPolicy(bucketName, writeOnlyPolicy)" functionAll += ", " + function writeOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` args = map[string]interface{}{ "bucketName": bucketName, "bucketPolicy": writeOnlyPolicy, } err = c.SetBucketPolicy(context.Background(), bucketName, writeOnlyPolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // should return policy `writeonly`. function = "GetBucketPolicy(ctx, bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } _, err = c.GetBucketPolicy(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return } // Make the bucket 'public read/write'. function = "SetBucketPolicy(bucketName, readWritePolicy)" functionAll += ", " + function readWritePolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` args = map[string]interface{}{ "bucketName": bucketName, "bucketPolicy": readWritePolicy, } err = c.SetBucketPolicy(context.Background(), bucketName, readWritePolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // should return policy `readwrite`. function = "GetBucketPolicy(bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } _, err = c.GetBucketPolicy(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return } // List all buckets. function = "ListBuckets()" functionAll += ", " + function args = nil buckets, err := c.ListBuckets(context.Background()) if len(buckets) == 0 { logError(testName, function, args, startTime, "", "Found bucket list to be empty", err) return } if err != nil { logError(testName, function, args, startTime, "", "ListBuckets failed", err) return } // Verify if previously created bucket is listed in list buckets. bucketFound := false for _, bucket := range buckets { if bucket.Name == bucketName { bucketFound = true } } // If bucket not found error out. if !bucketFound { logError(testName, function, args, startTime, "", "Bucket: "+bucketName+" not found", err) return } objectName := bucketName + "unique" // Generate data buf := bytes.Repeat([]byte("f"), 1<<19) function = "PutObject(bucketName, objectName, reader, contentType)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "contentType": "", } _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName + "-nolength", "contentType": "binary/octet-stream", } _, err = c.PutObject(context.Background(), bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Instantiate a done channel to close all listing. doneCh := make(chan struct{}) defer close(doneCh) objFound := false isRecursive := true // Recursive is true. function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "isRecursive": isRecursive, } for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Prefix: objectName, Recursive: true}) { if obj.Key == objectName { objFound = true break } } if !objFound { logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err) return } objFound = false isRecursive = true // Recursive is true. function = "ListObjects()" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "isRecursive": isRecursive, } for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{Prefix: objectName, Recursive: isRecursive}) { if obj.Key == objectName { objFound = true break } } if !objFound { logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err) return } incompObjNotFound := true function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "isRecursive": isRecursive, } for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive) { if objIncompl.Key != "" { incompObjNotFound = false break } } if !incompObjNotFound { logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err) return } function = "GetObject(bucketName, objectName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, } newReader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } newReadBytes, err := io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } if !bytes.Equal(newReadBytes, buf) { logError(testName, function, args, startTime, "", "GetObject bytes mismatch", err) return } newReader.Close() function = "FGetObject(bucketName, objectName, fileName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "fileName": fileName + "-f", } err = c.FGetObject(context.Background(), bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FGetObject failed", err) return } function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": "", "expires": 3600 * time.Second, } if _, err = c.PresignedHeadObject(context.Background(), bucketName, "", 3600*time.Second, nil); err == nil { logError(testName, function, args, startTime, "", "PresignedHeadObject success", err) return } // Generate presigned HEAD object url. function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "expires": 3600 * time.Second, } presignedHeadURL, err := c.PresignedHeadObject(context.Background(), bucketName, objectName, 3600*time.Second, nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) return } transport := createHTTPTransport() if err != nil { logError(testName, function, args, startTime, "", "DefaultTransport failed", err) return } httpClient := &http.Client{ // Setting a sensible time out of 30secs to wait for response // headers. Request is pro-actively canceled after 30secs // with no response. Timeout: 30 * time.Second, Transport: transport, } req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject request was incorrect", err) return } // Verify if presigned url works. resp, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err) return } if resp.StatusCode != http.StatusOK { logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect, status "+string(resp.StatusCode), err) return } if resp.Header.Get("ETag") == "" { logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err) return } resp.Body.Close() function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": "", "expires": 3600 * time.Second, } _, err = c.PresignedGetObject(context.Background(), bucketName, "", 3600*time.Second, nil) if err == nil { logError(testName, function, args, startTime, "", "PresignedGetObject success", err) return } // Generate presigned GET object url. function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "expires": 3600 * time.Second, } presignedGetURL, err := c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) return } // Verify if presigned url works. req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) return } resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } if resp.StatusCode != http.StatusOK { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err) return } newPresignedBytes, err := io.ReadAll(resp.Body) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } resp.Body.Close() if !bytes.Equal(newPresignedBytes, buf) { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } // Set request parameters. reqParams := make(url.Values) reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "expires": 3600 * time.Second, "reqParams": reqParams, } presignedGetURL, err = c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, reqParams) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) return } // Verify if presigned url works. req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) return } resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } if resp.StatusCode != http.StatusOK { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err) return } newPresignedBytes, err = io.ReadAll(resp.Body) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } if !bytes.Equal(newPresignedBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch for presigned GET URL", err) return } if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { logError(testName, function, args, startTime, "", "wrong Content-Disposition received "+string(resp.Header.Get("Content-Disposition")), err) return } function = "PresignedPutObject(bucketName, objectName, expires)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": "", "expires": 3600 * time.Second, } _, err = c.PresignedPutObject(context.Background(), bucketName, "", 3600*time.Second) if err == nil { logError(testName, function, args, startTime, "", "PresignedPutObject success", err) return } function = "PresignedPutObject(bucketName, objectName, expires)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName + "-presigned", "expires": 3600 * time.Second, } presignedPutURL, err := c.PresignedPutObject(context.Background(), bucketName, objectName+"-presigned", 3600*time.Second) if err != nil { logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) return } buf = bytes.Repeat([]byte("g"), 1<<19) req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf)) if err != nil { logError(testName, function, args, startTime, "", "Couldn't make HTTP request with PresignedPutObject URL", err) return } resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) return } newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presigned", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject after PresignedPutObject failed", err) return } newReadBytes, err = io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll after GetObject failed", err) return } if !bytes.Equal(newReadBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch", err) return } function = "PresignHeader(method, bucketName, objectName, expires, reqParams, extraHeaders)" functionAll += ", " + function presignExtraHeaders := map[string][]string{ "mysecret": {"abcxxx"}, } args = map[string]interface{}{ "method": "PUT", "bucketName": bucketName, "objectName": objectName + "-presign-custom", "expires": 3600 * time.Second, "extraHeaders": presignExtraHeaders, } presignedURL, err := c.PresignHeader(context.Background(), "PUT", bucketName, objectName+"-presign-custom", 3600*time.Second, nil, presignExtraHeaders) if err != nil { logError(testName, function, args, startTime, "", "Presigned failed", err) return } // Generate data more than 32K buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024) req, err = http.NewRequest(http.MethodPut, presignedURL.String(), bytes.NewReader(buf)) if err != nil { logError(testName, function, args, startTime, "", "HTTP request to Presigned URL failed", err) return } req.Header.Add("mysecret", "abcxxx") resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "HTTP request to Presigned URL failed", err) return } // Download the uploaded object to verify args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName + "-presign-custom", } newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presign-custom", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject of uploaded custom-presigned object failed", err) return } newReadBytes, err = io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed during get on custom-presigned put object", err) return } newReader.Close() if !bytes.Equal(newReadBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch on custom-presigned object upload verification", err) return } function = "RemoveObject(bucketName, objectName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, } err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) return } args["objectName"] = objectName + "-f" err = c.RemoveObject(context.Background(), bucketName, objectName+"-f", minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) return } args["objectName"] = objectName + "-nolength" err = c.RemoveObject(context.Background(), bucketName, objectName+"-nolength", minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) return } args["objectName"] = objectName + "-presigned" err = c.RemoveObject(context.Background(), bucketName, objectName+"-presigned", minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) return } args["objectName"] = objectName + "-presign-custom" err = c.RemoveObject(context.Background(), bucketName, objectName+"-presign-custom", minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) return } function = "RemoveBucket(bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } err = c.RemoveBucket(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "RemoveBucket failed", err) return } err = c.RemoveBucket(context.Background(), bucketName) if err == nil { logError(testName, function, args, startTime, "", "RemoveBucket did not fail for invalid bucket name", err) return } if err.Error() != "The specified bucket does not exist" { logError(testName, function, args, startTime, "", "RemoveBucket failed", err) return } os.Remove(fileName) os.Remove(fileName + "-f") logSuccess(testName, functionAll, args, startTime) } // Test for validating GetObject Reader* methods functioning when the // object is modified in the object store. func testGetObjectModified() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Upload an object. objectName := "myobject" args["objectName"] = objectName content := "helloworld" _, err = c.PutObject(context.Background(), bucketName, objectName, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "application/text"}) if err != nil { logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) return } defer c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{}) reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Failed to GetObject "+objectName+", from bucket "+bucketName, err) return } defer reader.Close() // Read a few bytes of the object. b := make([]byte, 5) n, err := reader.ReadAt(b, 0) if err != nil { logError(testName, function, args, startTime, "", "Failed to read object "+objectName+", from bucket "+bucketName+" at an offset", err) return } // Upload different contents to the same object while object is being read. newContent := "goodbyeworld" _, err = c.PutObject(context.Background(), bucketName, objectName, strings.NewReader(newContent), int64(len(newContent)), minio.PutObjectOptions{ContentType: "application/text"}) if err != nil { logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) return } // Confirm that a Stat() call in between doesn't change the Object's cached etag. _, err = reader.Stat() expectedError := "At least one of the pre-conditions you specified did not hold." if err.Error() != expectedError { logError(testName, function, args, startTime, "", "Expected Stat to fail with error "+expectedError+", but received "+err.Error(), err) return } // Read again only to find object contents have been modified since last read. _, err = reader.ReadAt(b, int64(n)) if err.Error() != expectedError { logError(testName, function, args, startTime, "", "Expected ReadAt to fail with error "+expectedError+", but received "+err.Error(), err) return } logSuccess(testName, function, args, startTime) } // Test validates putObject to upload a file seeked at a given offset. func testPutObjectUploadSeekedObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, fileToUpload, contentType)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "fileToUpload": "", "contentType": "binary/octet-stream", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) var tempfile *os.File if fileName := getMintDataDirFilePath("datafile-100-kB"); fileName != "" { tempfile, err = os.Open(fileName) if err != nil { logError(testName, function, args, startTime, "", "File open failed", err) return } args["fileToUpload"] = fileName } else { tempfile, err = os.CreateTemp("", "minio-go-upload-test-") if err != nil { logError(testName, function, args, startTime, "", "TempFile create failed", err) return } args["fileToUpload"] = tempfile.Name() // Generate 100kB data if _, err = io.Copy(tempfile, getDataReader("datafile-100-kB")); err != nil { logError(testName, function, args, startTime, "", "File copy failed", err) return } defer os.Remove(tempfile.Name()) // Seek back to the beginning of the file. tempfile.Seek(0, 0) } length := 100 * humanize.KiByte objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) args["objectName"] = objectName offset := length / 2 if _, err = tempfile.Seek(int64(offset), 0); err != nil { logError(testName, function, args, startTime, "", "TempFile seek failed", err) return } _, err = c.PutObject(context.Background(), bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } tempfile.Close() obj, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer obj.Close() n, err := obj.Seek(int64(offset), 0) if err != nil { logError(testName, function, args, startTime, "", "Seek failed", err) return } if n != int64(offset) { logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(offset), n), err) return } _, err = c.PutObject(context.Background(), bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName+"getobject", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if st.Size != int64(length-offset) { logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(length-offset), n), err) return } logSuccess(testName, function, args, startTime) } // Tests bucket re-create errors. func testMakeBucketErrorV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "MakeBucket(bucketName, region)" args := map[string]interface{}{ "bucketName": "", "region": "eu-west-1", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") region := "eu-west-1" args["bucketName"] = bucketName args["region"] = region // Make a new bucket in 'eu-west-1'. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err == nil { logError(testName, function, args, startTime, "", "MakeBucket did not fail for existing bucket name", err) return } // Verify valid error response from server. if minio.ToErrorResponse(err).Code != minio.BucketAlreadyExists && minio.ToErrorResponse(err).Code != minio.BucketAlreadyOwnedByYou { logError(testName, function, args, startTime, "", "Invalid error returned by server", err) return } logSuccess(testName, function, args, startTime) } // Test get object reader to not throw error on being closed twice. func testGetObjectClosedTwiceV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "MakeBucket(bucketName, region)" args := map[string]interface{}{ "bucketName": "", "region": "eu-west-1", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if err := r.Close(); err == nil { logError(testName, function, args, startTime, "", "Object is already closed, should return error", err) return } logSuccess(testName, function, args, startTime) } // Tests FPutObject hidden contentType setting func testFPutObjectV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FPutObject(bucketName, objectName, fileName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "fileName": "", "opts": "", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Make a temp file with 11*1024*1024 bytes of data. file, err := os.CreateTemp(os.TempDir(), "FPutObjectTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return } r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024)) n, err := io.CopyN(file, r, 11*1024*1024) if err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } if n != int64(11*1024*1024) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) return } // Close the file pro-actively for windows. err = file.Close() if err != nil { logError(testName, function, args, startTime, "", "File close failed", err) return } // Set base object name objectName := bucketName + "FPutObject" args["objectName"] = objectName args["fileName"] = file.Name() // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) _, err = c.FPutObject(context.Background(), bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } // Perform FPutObject with no contentType provided (Expecting application/octet-stream) args["objectName"] = objectName + "-Octet" args["contentType"] = "" _, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } // Add extension to temp file name fileName := file.Name() err = os.Rename(fileName, fileName+".gtar") if err != nil { logError(testName, function, args, startTime, "", "Rename failed", err) return } // Perform FPutObject with no contentType provided (Expecting application/x-gtar) args["objectName"] = objectName + "-Octet" args["contentType"] = "" args["fileName"] = fileName + ".gtar" _, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } // Check headers and sizes rStandard, err := c.StatObject(context.Background(), bucketName, objectName+"-standard", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if rStandard.Size != 11*1024*1024 { logError(testName, function, args, startTime, "", "Unexpected size", nil) return } if rStandard.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rStandard.ContentType, err) return } rOctet, err := c.StatObject(context.Background(), bucketName, objectName+"-Octet", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if rOctet.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rOctet.ContentType, err) return } if rOctet.Size != 11*1024*1024 { logError(testName, function, args, startTime, "", "Unexpected size", nil) return } rGTar, err := c.StatObject(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if rGTar.Size != 11*1024*1024 { logError(testName, function, args, startTime, "", "Unexpected size", nil) return } if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" && rGTar.ContentType != "application/x-tar" { logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-tar , got "+rGTar.ContentType, err) return } os.Remove(fileName + ".gtar") logSuccess(testName, function, args, startTime) } // Tests various bucket supported formats. func testMakeBucketRegionsV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "MakeBucket(bucketName, region)" args := map[string]interface{}{ "bucketName": "", "region": "eu-west-1", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket in 'eu-central-1'. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "eu-west-1"}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed while removing bucket recursively", err) return } // Make a new bucket with '.' in its name, in 'us-west-2'. This // request is internally staged into a path style instead of // virtual host style. if err = c.MakeBucket(context.Background(), bucketName+".withperiod", minio.MakeBucketOptions{Region: "us-west-2"}); err != nil { args["bucketName"] = bucketName + ".withperiod" args["region"] = "us-west-2" logError(testName, function, args, startTime, "", "MakeBucket test with a bucket name with period, '.', failed", err) return } // Delete all objects and buckets if err = cleanupBucket(bucketName+".withperiod", c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed while removing bucket recursively", err) return } logSuccess(testName, function, args, startTime) } // Tests get object ReaderSeeker interface methods. func testGetObjectReadSeekFunctionalV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer r.Close() st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) return } offset := int64(2048) n, err := r.Seek(offset, 0) if err != nil { logError(testName, function, args, startTime, "", "Seek failed", err) return } if n != offset { logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) return } n, err = r.Seek(0, 1) if err != nil { logError(testName, function, args, startTime, "", "Seek failed", err) return } if n != offset { logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) return } _, err = r.Seek(offset, 2) if err == nil { logError(testName, function, args, startTime, "", "Seek on positive offset for whence '2' should error out", err) return } n, err = r.Seek(-offset, 2) if err != nil { logError(testName, function, args, startTime, "", "Seek failed", err) return } if n != st.Size-offset { logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(st.Size-offset)+" got "+string(n), err) return } var buffer1 bytes.Buffer if _, err = io.CopyN(&buffer1, r, st.Size); err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "Copy failed", err) return } } if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) { logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) return } // Seek again and read again. n, err = r.Seek(offset-1, 0) if err != nil { logError(testName, function, args, startTime, "", "Seek failed", err) return } if n != (offset - 1) { logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset-1)+" got "+string(n), err) return } var buffer2 bytes.Buffer if _, err = io.CopyN(&buffer2, r, st.Size); err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "Copy failed", err) return } } // Verify now lesser bytes. if !bytes.Equal(buf[2047:], buffer2.Bytes()) { logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) return } logSuccess(testName, function, args, startTime) } // Tests get object ReaderAt interface methods. func testGetObjectReadAtFunctionalV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer r.Close() st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) return } offset := int64(2048) // Read directly buf2 := make([]byte, 512) buf3 := make([]byte, 512) buf4 := make([]byte, 512) m, err := r.ReadAt(buf2, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf2) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+" got "+string(m), err) return } if !bytes.Equal(buf2, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf3, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf3) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+" got "+string(m), err) return } if !bytes.Equal(buf3, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf4, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf4) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+" got "+string(m), err) return } if !bytes.Equal(buf4, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } buf5 := make([]byte, bufSize) // Read the whole object. m, err = r.ReadAt(buf5, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } if m != len(buf5) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+" got "+string(m), err) return } if !bytes.Equal(buf, buf5) { logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) return } buf6 := make([]byte, bufSize+1) // Read the whole object and beyond. _, err = r.ReadAt(buf6, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } logSuccess(testName, function, args, startTime) } // Tests copy object func testCopyObjectV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Make a new bucket in 'us-east-1' (destination bucket). err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName+"-copy", c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } // Check the various fields of source object against destination object. objInfo, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } r.Close() // Copy Source src := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), MatchETag: objInfo.ETag, } args["source"] = src // Set copy conditions. dst := minio.CopyDestOptions{ Bucket: bucketName + "-copy", Object: objectName + "-copy", } args["destination"] = dst // Perform the Copy _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Source object r, err = c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } // Destination object readerCopy, err := c.GetObject(context.Background(), bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } // Check the various fields of source object against destination object. objInfo, err = r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } objInfoCopy, err := readerCopy.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if objInfo.Size != objInfoCopy.Size { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+" got "+string(objInfo.Size), err) return } // Close all the readers. r.Close() readerCopy.Close() // CopyObject again but with wrong conditions src = minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, MatchUnmodifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), NoMatchETag: objInfo.ETag, } // Perform the Copy which should fail _, err = c.CopyObject(context.Background(), dst, src) if err == nil { logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) return } logSuccess(testName, function, args, startTime) } // Tests copy object with various checksum scenarios, tries to not repeat CopyObjectV2 test and // instead just focus on Checksum. func testCopyObjectWithChecksums() { startTime := time.Now() testName := getFuncName() function := "CopyObjectWithChecksums(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Make a new bucket in 'us-east-1' (destination bucket). err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName+"-copy", c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // PutObject to upload the object to the bucket, this object will have a Crc64NVME checksum applied // by default since nothing was explicitly specified. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // GetObject to obtain the eTag r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } objInfo, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } r.Close() // Copy source options src := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), MatchETag: objInfo.ETag, } tests := []struct { csType minio.ChecksumType cs wantChecksums }{ {csType: minio.ChecksumCRC64NVME, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}}, {csType: minio.ChecksumCRC32C, cs: wantChecksums{minio.ChecksumCRC32C: "aHnJMw=="}}, {csType: minio.ChecksumCRC32, cs: wantChecksums{minio.ChecksumCRC32: "tIZ8hA=="}}, {csType: minio.ChecksumSHA1, cs: wantChecksums{minio.ChecksumSHA1: "6YIIbcWH1iLaCFqs5vwq5Rwvm+o="}}, {csType: minio.ChecksumSHA256, cs: wantChecksums{minio.ChecksumSHA256: "GKeJTopbMGPs3h4fAw4oe0R2QnnmFVJeIWkqCkp28Yo="}}, // In S3, all copied objects without checksums and specified destination checksum algorithms // automatically gain a CRC-64NVME checksum algorithm. Use ChecksumNone for this case. {csType: minio.ChecksumNone, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}}, } for _, test := range tests { args := map[string]interface{}{} args["srcOpts"] = src args["section"] = "setup" args["checksum"] = test.csType.String() // Copy destination options bucketCopyName := bucketName + "-copy" objectCopyName := objectName + "-copy-" + test.csType.String() dst := minio.CopyDestOptions{ Bucket: bucketCopyName, Object: objectCopyName, ReplaceMetadata: true, } if test.csType != minio.ChecksumNone { // Request the server-side checksum on the copy. // ChecksumNone is a flag to leave off the header dst.ChecksumType = test.csType } args["destOpts"] = dst // Perform the Copy args["section"] = "CopyObject" _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Checksum verification args["section"] = "HeadObject" st, err := c.StatObject(context.Background(), bucketCopyName, objectCopyName, minio.StatObjectOptions{Checksum: true}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if st.ChecksumMode != "FULL_OBJECT" { logError(testName, function, args, startTime, "", "ChecksumMode want: FULL_OBJECT, got "+st.ChecksumMode, nil) return } err = cmpChecksum(st, test.cs) if err != nil { logError(testName, function, args, startTime, "", "Checksum mismatch", err) return } logSuccess(testName, function, args, startTime) } } // Tests replacing an object with CopyObject and a new Checksum type func testReplaceObjectWithChecksums() { startTime := time.Now() testName := getFuncName() function := "CopyObjectWithChecksums(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) tests := []struct { csType minio.ChecksumType cs wantChecksums }{ {csType: minio.ChecksumCRC64NVME, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}}, {csType: minio.ChecksumCRC32C, cs: wantChecksums{minio.ChecksumCRC32C: "aHnJMw=="}}, {csType: minio.ChecksumCRC32, cs: wantChecksums{minio.ChecksumCRC32: "tIZ8hA=="}}, {csType: minio.ChecksumSHA1, cs: wantChecksums{minio.ChecksumSHA1: "6YIIbcWH1iLaCFqs5vwq5Rwvm+o="}}, {csType: minio.ChecksumSHA256, cs: wantChecksums{minio.ChecksumSHA256: "GKeJTopbMGPs3h4fAw4oe0R2QnnmFVJeIWkqCkp28Yo="}}, // In S3, all copied objects without checksums and specified destination checksum algorithms // automatically gain a CRC-64NVME checksum algorithm. Use ChecksumNone for this case. {csType: minio.ChecksumNone, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}}, } for _, test := range tests { args := map[string]interface{}{} args["section"] = "setup" args["destOpts"] = "" args["checksum"] = test.csType.String() bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // PutObject to upload the object to the bucket objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // GetObject to obtain the eTag r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } objInfo, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } r.Close() // Copy source options src := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), MatchETag: objInfo.ETag, } // Copy destination options, overwrite the existing object dst := minio.CopyDestOptions{ Bucket: bucketName, Object: objectName, // S3 requires that we send some new metadata otherwise it complains that the // CopyObject is illegal. UserMetadata: map[string]string{ "TestMeta": objectName + "-meta-" + test.csType.String(), }, ReplaceMetadata: true, } if test.csType != minio.ChecksumNone { // Request the server-side checksum on the copy. // ChecksumNone is a flag to leave off the header dst.ChecksumType = test.csType } args["destOpts"] = dst // Perform the Copy args["section"] = "CopyObject" _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Checksum verification args["section"] = "HeadObject" st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{Checksum: true}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if st.ChecksumMode != "FULL_OBJECT" { logError(testName, function, args, startTime, "", "ChecksumMode want: FULL_OBJECT, got "+st.ChecksumMode, nil) return } err = cmpChecksum(st, test.cs) if err != nil { logError(testName, function, args, startTime, "", "Checksum mismatch", err) return } logSuccess(testName, function, args, startTime) } } func testComposeObjectErrorCasesWrapper(c *minio.Client) { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ComposeObject(destination, sourceList)" args := map[string]interface{}{} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Test that more than 10K source objects cannot be // concatenated. srcArr := [10001]minio.CopySrcOptions{} srcSlice := srcArr[:] dst := minio.CopyDestOptions{ Bucket: bucketName, Object: "object", } args["destination"] = dst // Just explain about srcArr in args["sourceList"] // to stop having 10,001 null headers logged args["sourceList"] = "source array of 10,001 elements" if _, err := c.ComposeObject(context.Background(), dst, srcSlice...); err == nil { logError(testName, function, args, startTime, "", "Expected error in ComposeObject", err) return } else if err.Error() != "There must be as least one and up to 10000 source objects." { logError(testName, function, args, startTime, "", "Got unexpected error", err) return } // Create a source with invalid offset spec and check that // error is returned: // 1. Create the source object. const badSrcSize = 5 * 1024 * 1024 buf := bytes.Repeat([]byte("1"), badSrcSize) _, err = c.PutObject(context.Background(), bucketName, "badObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // 2. Set invalid range spec on the object (going beyond // object size) badSrc := minio.CopySrcOptions{ Bucket: bucketName, Object: "badObject", MatchRange: true, Start: 1, End: badSrcSize, } // 3. ComposeObject call should fail. if _, err := c.ComposeObject(context.Background(), dst, badSrc); err == nil { logError(testName, function, args, startTime, "", "ComposeObject expected to fail", err) return } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { logError(testName, function, args, startTime, "", "Got invalid error", err) return } logSuccess(testName, function, args, startTime) } // Test expected error cases func testComposeObjectErrorCasesV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ComposeObject(destination, sourceList)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } testComposeObjectErrorCasesWrapper(c) } func testComposeMultipleSources(c *minio.Client) { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ComposeObject(destination, sourceList)" args := map[string]interface{}{ "destination": "", "sourceList": "", } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Upload a small source object const srcSize = 1024 * 1024 * 5 buf := bytes.Repeat([]byte("1"), srcSize) _, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(srcSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // We will append 10 copies of the object. srcs := []minio.CopySrcOptions{} for i := 0; i < 10; i++ { srcs = append(srcs, minio.CopySrcOptions{ Bucket: bucketName, Object: "srcObject", }) } // make the last part very small srcs[9].MatchRange = true args["sourceList"] = srcs dst := minio.CopyDestOptions{ Bucket: bucketName, Object: "dstObject", } args["destination"] = dst ui, err := c.ComposeObject(context.Background(), dst, srcs...) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return } if ui.Size != 9*srcSize+1 { logError(testName, function, args, startTime, "", "ComposeObject returned unexpected size", err) return } objProps, err := c.StatObject(context.Background(), bucketName, "dstObject", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if objProps.Size != 9*srcSize+1 { logError(testName, function, args, startTime, "", "Size mismatched! Expected "+string(10000*srcSize)+" got "+string(objProps.Size), err) return } logSuccess(testName, function, args, startTime) } // Test concatenating multiple 10K objects V2 func testCompose10KSourcesV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ComposeObject(destination, sourceList)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } testComposeMultipleSources(c) } func testEncryptedEmptyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, objectSize, opts)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) sse := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"object")) // 1. create an sse-c encrypted object to copy by uploading const srcSize = 0 var buf []byte // Empty buffer args["objectName"] = "object" _, err = c.PutObject(context.Background(), bucketName, "object", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } // 2. Test CopyObject for an empty object src := minio.CopySrcOptions{ Bucket: bucketName, Object: "object", Encryption: sse, } dst := minio.CopyDestOptions{ Bucket: bucketName, Object: "new-object", Encryption: sse, } if _, err = c.CopyObject(context.Background(), dst, src); err != nil { function = "CopyObject(dst, src)" logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject failed", err) return } // 3. Test Key rotation newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"new-object")) src = minio.CopySrcOptions{ Bucket: bucketName, Object: "new-object", Encryption: sse, } dst = minio.CopyDestOptions{ Bucket: bucketName, Object: "new-object", Encryption: newSSE, } if _, err = c.CopyObject(context.Background(), dst, src); err != nil { function = "CopyObject(dst, src)" logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject with key rotation failed", err) return } // 4. Download the object. reader, err := c.GetObject(context.Background(), bucketName, "new-object", minio.GetObjectOptions{ServerSideEncryption: newSSE}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer reader.Close() decBytes, err := io.ReadAll(reader) if err != nil { logError(testName, function, map[string]interface{}{}, startTime, "", "ReadAll failed", err) return } if !bytes.Equal(decBytes, buf) { logError(testName, function, map[string]interface{}{}, startTime, "", "Downloaded object doesn't match the empty encrypted object", err) return } delete(args, "objectName") logSuccess(testName, function, args, startTime) } func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, sseDst encrypt.ServerSide) { // initialize logging params startTime := time.Now() testName := getFuncNameLoc(2) function := "CopyObject(destination, source)" args := map[string]interface{}{} args["testName"] = testName var srcEncryption, dstEncryption encrypt.ServerSide // Make a new bucket in 'us-east-1' (source bucket). err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // 1. create an sse-c encrypted object to copy by uploading const srcSize = 1024 * 1024 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB // Calculate the CRC32C checksum for the object meta := map[string]string{} h := minio.ChecksumCRC32C.Hasher() h.Reset() h.Write(buf) meta[minio.ChecksumCRC32C.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil)) _, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ServerSideEncryption: sseSrc, DisableMultipart: true, DisableContentSha256: true, UserMetadata: meta, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } if sseSrc != nil && sseSrc.Type() != encrypt.S3 { srcEncryption = sseSrc } // 2. copy object and change encryption key src := minio.CopySrcOptions{ Bucket: bucketName, Object: "srcObject", Encryption: srcEncryption, } args["source"] = src dst := minio.CopyDestOptions{ Bucket: bucketName, Object: "dstObject", Encryption: sseDst, } args["destination"] = dst _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } if sseDst != nil && sseDst.Type() != encrypt.S3 { dstEncryption = sseDst } // 3. get copied object and check if content is equal coreClient := minio.Core{Client: c} reader, oi, _, err := coreClient.GetObject(context.Background(), bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption, Checksum: true}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } decBytes, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } if !bytes.Equal(decBytes, buf) { logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) return } reader.Close() err = cmpChecksum(oi, wantChecksums{minio.ChecksumCRC32C: "bSoobA=="}) if err != nil { logError(testName, function, args, startTime, "", "Checksum mismatch on dstObject", err) return } // Test key rotation for source object in-place. var newSSE encrypt.ServerSide if sseSrc != nil && sseSrc.Type() == encrypt.SSEC { newSSE = encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"srcObject")) // replace key } if sseSrc != nil && sseSrc.Type() == encrypt.S3 { newSSE = encrypt.NewSSE() } if newSSE != nil { dst = minio.CopyDestOptions{ Bucket: bucketName, Object: "srcObject", Encryption: newSSE, } args["destination"] = dst _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Get copied object and check if content is equal reader, oi, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE, Checksum: true}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } decBytes, err = io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } if !bytes.Equal(decBytes, buf) { logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) return } reader.Close() err = cmpChecksum(oi, wantChecksums{minio.ChecksumCRC32C: "bSoobA=="}) if err != nil { fmt.Printf("srcObject objectInfo: %+v\n", oi) logError(testName, function, args, startTime, "", "Checksum mismatch on srcObject for in-place", err) return } // Test in-place decryption. dst = minio.CopyDestOptions{ Bucket: bucketName, Object: "srcObject", } args["destination"] = dst src = minio.CopySrcOptions{ Bucket: bucketName, Object: "srcObject", Encryption: newSSE, } args["source"] = src _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject Key rotation failed", err) return } } // Get copied decrypted object and check if content is equal reader, oi, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{Checksum: true}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer reader.Close() decBytes, err = io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } if !bytes.Equal(decBytes, buf) { logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) return } err = cmpChecksum(oi, wantChecksums{minio.ChecksumCRC32C: "bSoobA=="}) if err != nil { logError(testName, function, args, startTime, "", "Checksum mismatch for decrypted object", err) return } logSuccess(testName, function, args, startTime) } // Test encrypted copy object func testUnencryptedToSSECCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) testEncryptedCopyObjectWrapper(c, bucketName, nil, sseDst) } // Test encrypted copy object func testUnencryptedToSSES3CopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") var sseSrc encrypt.ServerSide sseDst := encrypt.NewSSE() testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testUnencryptedToUnencryptedCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") var sseSrc, sseDst encrypt.ServerSide testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testEncryptedSSECToSSECCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testEncryptedSSECToSSES3CopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) sseDst := encrypt.NewSSE() testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testEncryptedSSECToUnencryptedCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) var sseDst encrypt.ServerSide testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testEncryptedSSES3ToSSECCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseSrc := encrypt.NewSSE() sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testEncryptedSSES3ToSSES3CopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseSrc := encrypt.NewSSE() sseDst := encrypt.NewSSE() testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testEncryptedSSES3ToUnencryptedCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseSrc := encrypt.NewSSE() var sseDst encrypt.ServerSide testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testEncryptedCopyObjectV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true, TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } func testDecryptedCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } bucketName, objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-"), "object" if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) encryption := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)) _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(bytes.Repeat([]byte("a"), 1024*1024)), 1024*1024, minio.PutObjectOptions{ ServerSideEncryption: encryption, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } src := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, Encryption: encrypt.SSECopy(encryption), } args["source"] = src dst := minio.CopyDestOptions{ Bucket: bucketName, Object: "decrypted-" + objectName, } args["destination"] = dst if _, err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } if _, err = c.GetObject(context.Background(), bucketName, "decrypted-"+objectName, minio.GetObjectOptions{}); err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } logSuccess(testName, function, args, startTime) } func testSSECMultipartEncryptedToSSECCopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 6MB of data buf := bytes.Repeat([]byte("abcdef"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) // Upload a 6MB object using multipart mechanism uploadID, err := c.NewMultipartUpload(context.Background(), bucketName, objectName, minio.PutObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } var completeParts []minio.CompletePart part, err := c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024, minio.PutObjectPartOptions{SSE: srcencryption}, ) if err != nil { logError(testName, function, args, startTime, "", "PutObjectPart call failed", err) return } completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) part, err = c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 2, bytes.NewReader(buf[5*1024*1024:]), 1024*1024, minio.PutObjectPartOptions{SSE: srcencryption}, ) if err != nil { logError(testName, function, args, startTime, "", "PutObjectPart call failed", err) return } completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, completeParts, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } destBucketName := bucketName destObjectName := objectName + "-dest" dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) uploadID, err = c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) encrypt.SSECopy(srcencryption).Marshal(header) dstencryption.Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = objInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (6*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 6*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 6*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 6MB", err) return } getOpts.SetRange(6*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 6*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:6*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 6MB", err) return } if getBuf[6*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation func testSSECEncryptedToSSECCopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) putmetadata := map[string]string{ "Content-Type": "binary/octet-stream", } opts := minio.PutObjectOptions{ UserMetadata: putmetadata, ServerSideEncryption: srcencryption, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) encrypt.SSECopy(srcencryption).Marshal(header) dstencryption.Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for SSEC encrypted to unencrypted copy func testSSECEncryptedToUnencryptedCopyPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) opts := minio.PutObjectOptions{ UserMetadata: map[string]string{ "Content-Type": "binary/octet-stream", }, ServerSideEncryption: srcencryption, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" var dstencryption encrypt.ServerSide uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) encrypt.SSECopy(srcencryption).Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for SSEC encrypted to SSE-S3 encrypted copy func testSSECEncryptedToSSES3CopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) putmetadata := map[string]string{ "Content-Type": "binary/octet-stream", } opts := minio.PutObjectOptions{ UserMetadata: putmetadata, ServerSideEncryption: srcencryption, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" dstencryption := encrypt.NewSSE() uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) encrypt.SSECopy(srcencryption).Marshal(header) dstencryption.Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for unencrypted to SSEC encryption copy part func testUnencryptedToSSECCopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" putmetadata := map[string]string{ "Content-Type": "binary/octet-stream", } opts := minio.PutObjectOptions{ UserMetadata: putmetadata, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) dstencryption.Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for unencrypted to unencrypted copy func testUnencryptedToUnencryptedCopyPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") putmetadata := map[string]string{ "Content-Type": "binary/octet-stream", } opts := minio.PutObjectOptions{ UserMetadata: putmetadata, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy func testUnencryptedToSSES3CopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") opts := minio.PutObjectOptions{ UserMetadata: map[string]string{ "Content-Type": "binary/octet-stream", }, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" dstencryption := encrypt.NewSSE() uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) dstencryption.Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for SSE-S3 to SSEC encryption copy part func testSSES3EncryptedToSSECCopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" srcEncryption := encrypt.NewSSE() opts := minio.PutObjectOptions{ UserMetadata: map[string]string{ "Content-Type": "binary/octet-stream", }, ServerSideEncryption: srcEncryption, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) dstencryption.Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for unencrypted to unencrypted copy func testSSES3EncryptedToUnencryptedCopyPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") srcEncryption := encrypt.NewSSE() opts := minio.PutObjectOptions{ UserMetadata: map[string]string{ "Content-Type": "binary/octet-stream", }, ServerSideEncryption: srcEncryption, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy func testSSES3EncryptedToSSES3CopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") srcEncryption := encrypt.NewSSE() opts := minio.PutObjectOptions{ UserMetadata: map[string]string{ "Content-Type": "binary/octet-stream", }, ServerSideEncryption: srcEncryption, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" dstencryption := encrypt.NewSSE() uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) dstencryption.Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } func testUserMetadataCopying() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } testUserMetadataCopyingWrapper(c) } func testUserMetadataCopyingWrapper(c *minio.Client) { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) fetchMeta := func(object string) (h http.Header) { objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return h } h = make(http.Header) for k, vs := range objInfo.Metadata { if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { h.Add(k, vs[0]) } } return h } // 1. create a client encrypted object to copy by uploading const srcSize = 1024 * 1024 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB metadata := make(http.Header) metadata.Set("x-amz-meta-myheader", "myvalue") m := make(map[string]string) m["x-amz-meta-myheader"] = "myvalue" _, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{UserMetadata: m}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectWithMetadata failed", err) return } if !reflect.DeepEqual(metadata, fetchMeta("srcObject")) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } // 2. create source src := minio.CopySrcOptions{ Bucket: bucketName, Object: "srcObject", } // 2.1 create destination with metadata set dst1 := minio.CopyDestOptions{ Bucket: bucketName, Object: "dstObject-1", UserMetadata: map[string]string{"notmyheader": "notmyvalue"}, ReplaceMetadata: true, } // 3. Check that copying to an object with metadata set resets // the headers on the copy. args["source"] = src args["destination"] = dst1 _, err = c.CopyObject(context.Background(), dst1, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } expectedHeaders := make(http.Header) expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-1")) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } // 4. create destination with no metadata set and same source dst2 := minio.CopyDestOptions{ Bucket: bucketName, Object: "dstObject-2", } // 5. Check that copying to an object with no metadata set, // copies metadata. args["source"] = src args["destination"] = dst2 _, err = c.CopyObject(context.Background(), dst2, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } expectedHeaders = metadata if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-2")) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } // 6. Compose a pair of sources. dst3 := minio.CopyDestOptions{ Bucket: bucketName, Object: "dstObject-3", ReplaceMetadata: true, } function = "ComposeObject(destination, sources)" args["source"] = []minio.CopySrcOptions{src, src} args["destination"] = dst3 _, err = c.ComposeObject(context.Background(), dst3, src, src) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return } // Check that no headers are copied in this case if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } // 7. Compose a pair of sources with dest user metadata set. dst4 := minio.CopyDestOptions{ Bucket: bucketName, Object: "dstObject-4", UserMetadata: map[string]string{"notmyheader": "notmyvalue"}, ReplaceMetadata: true, } function = "ComposeObject(destination, sources)" args["source"] = []minio.CopySrcOptions{src, src} args["destination"] = dst4 _, err = c.ComposeObject(context.Background(), dst4, src, src) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return } // Check that no headers are copied in this case expectedHeaders = make(http.Header) expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } logSuccess(testName, function, args, startTime) } func testUserMetadataCopyingV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } testUserMetadataCopyingWrapper(c) } func testStorageClassMetadataPutObject() { // initialize logging params startTime := time.Now() function := "testStorageClassMetadataPutObject()" args := map[string]interface{}{} testName := getFuncName() c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) fetchMeta := func(object string) (h http.Header) { objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return h } h = make(http.Header) for k, vs := range objInfo.Metadata { if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") { for _, v := range vs { h.Add(k, v) } } } return h } metadata := make(http.Header) metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") emptyMetadata := make(http.Header) const srcSize = 1024 * 1024 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Get the returned metadata returnedMeta := fetchMeta("srcObjectRRSClass") // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways) if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } metadata = make(http.Header) metadata.Set("x-amz-storage-class", "STANDARD") _, err = c.PutObject(context.Background(), bucketName, "srcObjectSSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClass")) { logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) return } logSuccess(testName, function, args, startTime) } func testStorageClassInvalidMetadataPutObject() { // initialize logging params startTime := time.Now() function := "testStorageClassInvalidMetadataPutObject()" args := map[string]interface{}{} testName := getFuncName() c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) const srcSize = 1024 * 1024 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "INVALID_STORAGE_CLASS"}) if err == nil { logError(testName, function, args, startTime, "", "PutObject with invalid storage class passed, was expected to fail", err) return } logSuccess(testName, function, args, startTime) } func testStorageClassMetadataCopyObject() { // initialize logging params startTime := time.Now() function := "testStorageClassMetadataCopyObject()" args := map[string]interface{}{} testName := getFuncName() c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) fetchMeta := func(object string) (h http.Header) { objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{}) args["bucket"] = bucketName args["object"] = object if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return h } h = make(http.Header) for k, vs := range objInfo.Metadata { if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") { for _, v := range vs { h.Add(k, v) } } } return h } metadata := make(http.Header) metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") emptyMetadata := make(http.Header) const srcSize = 1024 * 1024 buf := bytes.Repeat([]byte("abcde"), srcSize) // Put an object with RRS Storage class _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Make server side copy of object uploaded in previous step src := minio.CopySrcOptions{ Bucket: bucketName, Object: "srcObjectRRSClass", } dst := minio.CopyDestOptions{ Bucket: bucketName, Object: "srcObjectRRSClassCopy", } if _, err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed on RRS", err) return } // Get the returned metadata returnedMeta := fetchMeta("srcObjectRRSClassCopy") // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways) if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } metadata = make(http.Header) metadata.Set("x-amz-storage-class", "STANDARD") // Put an object with Standard Storage class _, err = c.PutObject(context.Background(), bucketName, "srcObjectSSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Make server side copy of object uploaded in previous step src = minio.CopySrcOptions{ Bucket: bucketName, Object: "srcObjectSSClass", } dst = minio.CopyDestOptions{ Bucket: bucketName, Object: "srcObjectSSClassCopy", } if _, err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed on SS", err) return } // Fetch the meta data of copied object if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClassCopy")) { logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) return } logSuccess(testName, function, args, startTime) } // Test put object with size -1 byte object. func testPutObjectNoLengthV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "size": -1, "opts": "", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) objectName := bucketName + "unique" args["objectName"] = objectName bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() args["size"] = bufSize // Upload an object. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, -1, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Expected upload object size "+string(bufSize)+" got "+string(st.Size), err) return } logSuccess(testName, function, args, startTime) } // Test put objects of unknown size. func testPutObjectsUnknownV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader,size,opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "size": "", "opts": "", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Issues are revealed by trying to upload multiple files of unknown size // sequentially (on 4GB machines) for i := 1; i <= 4; i++ { // Simulate that we could be receiving byte slices of data that we want // to upload as a file rpipe, wpipe := io.Pipe() defer rpipe.Close() go func() { b := []byte("test") wpipe.Write(b) wpipe.Close() }() // Upload the object. objectName := fmt.Sprintf("%sunique%d", bucketName, i) args["objectName"] = objectName ui, err := c.PutObject(context.Background(), bucketName, objectName, rpipe, -1, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) return } if ui.Size != 4 { logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(ui.Size), nil) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObjectStreaming failed", err) return } if st.Size != int64(4) { logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(st.Size), err) return } } logSuccess(testName, function, args, startTime) } // Test put object with 0 byte object. func testPutObject0ByteV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "size": 0, "opts": "", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) objectName := bucketName + "unique" args["objectName"] = objectName args["opts"] = minio.PutObjectOptions{} // Upload an object. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObjectWithSize failed", err) return } if st.Size != 0 { logError(testName, function, args, startTime, "", "Expected upload object size 0 but got "+string(st.Size), err) return } logSuccess(testName, function, args, startTime) } // Test put object with 0 byte object with non-US-ASCII characters. func testPutObjectMetadataNonUSASCIIV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "size": 0, "opts": "", } metadata := map[string]string{ "test-zh": "你好", "test-ja": "こんにちは", "test-ko": "안녕하세요", "test-ru": "Здравствуй", "test-de": "Hallo", "test-it": "Ciao", "test-pt": "Olá", "test-ar": "مرحبا", "test-hi": "नमस्ते", "test-hu": "Helló", "test-ro": "Bună", "test-be": "Прывiтанне", "test-sl": "Pozdravljen", "test-sr": "Здраво", "test-bg": "Здравейте", "test-uk": "Привіт", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) objectName := bucketName + "unique" args["objectName"] = objectName args["opts"] = minio.PutObjectOptions{} // Upload an object. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{ UserMetadata: metadata, }) if err != nil { logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObjectWithSize failed", err) return } if st.Size != 0 { logError(testName, function, args, startTime, "", "Expected upload object size 0 but got "+string(st.Size), err) return } for k, v := range metadata { if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") { continue } if st.Metadata.Get(http.CanonicalHeaderKey("X-Amz-Meta-"+k)) != v { logError(testName, function, args, startTime, "", "Expected upload object metadata "+k+": "+v+" but got "+st.Metadata.Get(http.CanonicalHeaderKey("X-Amz-Meta-"+k)), err) return } } logSuccess(testName, function, args, startTime) } // Test expected error cases func testComposeObjectErrorCases() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ComposeObject(destination, sourceList)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } testComposeObjectErrorCasesWrapper(c) } // Test concatenating multiple 10K objects V4 func testCompose10KSources() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ComposeObject(destination, sourceList)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } testComposeMultipleSources(c) } // Tests comprehensive list of all methods. func testFunctionalV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "testFunctionalV2()" functionAll := "" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") location := "us-east-1" // Make a new bucket. function = "MakeBucket(bucketName, location)" functionAll = "MakeBucket(bucketName, location)" args = map[string]interface{}{ "bucketName": bucketName, "location": location, } err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: location}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate a random file name. fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") file, err := os.Create(fileName) if err != nil { logError(testName, function, args, startTime, "", "file create failed", err) return } for i := 0; i < 3; i++ { buf := make([]byte, rand.Intn(1<<19)) _, err = file.Write(buf) if err != nil { logError(testName, function, args, startTime, "", "file write failed", err) return } } file.Close() // Verify if bucket exits and you have access. var exists bool function = "BucketExists(bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } exists, err = c.BucketExists(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "BucketExists failed", err) return } if !exists { logError(testName, function, args, startTime, "", "Could not find existing bucket "+bucketName, err) return } // Make the bucket 'public read/write'. function = "SetBucketPolicy(bucketName, bucketPolicy)" functionAll += ", " + function readWritePolicy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:ListBucketMultipartUploads", "s3:ListBucket"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::` + bucketName + `"],"Sid": ""}]}` args = map[string]interface{}{ "bucketName": bucketName, "bucketPolicy": readWritePolicy, } err = c.SetBucketPolicy(context.Background(), bucketName, readWritePolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // List all buckets. function = "ListBuckets()" functionAll += ", " + function args = nil buckets, err := c.ListBuckets(context.Background()) if len(buckets) == 0 { logError(testName, function, args, startTime, "", "List buckets cannot be empty", err) return } if err != nil { logError(testName, function, args, startTime, "", "ListBuckets failed", err) return } // Verify if previously created bucket is listed in list buckets. bucketFound := false for _, bucket := range buckets { if bucket.Name == bucketName { bucketFound = true } } // If bucket not found error out. if !bucketFound { logError(testName, function, args, startTime, "", "Bucket "+bucketName+"not found", err) return } objectName := bucketName + "unique" // Generate data buf := bytes.Repeat([]byte("n"), rand.Intn(1<<19)) args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "contentType": "", } _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(st.Size), err) return } objectNameNoLength := objectName + "-nolength" args["objectName"] = objectNameNoLength _, err = c.PutObject(context.Background(), bucketName, objectNameNoLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } st, err = c.StatObject(context.Background(), bucketName, objectNameNoLength, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(st.Size), err) return } // Instantiate a done channel to close all listing. doneCh := make(chan struct{}) defer close(doneCh) objFound := false isRecursive := true // Recursive is true. function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "isRecursive": isRecursive, } for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Prefix: objectName, Recursive: isRecursive}) { if obj.Key == objectName { objFound = true break } } if !objFound { logError(testName, function, args, startTime, "", "Could not find existing object "+objectName, err) return } incompObjNotFound := true function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "isRecursive": isRecursive, } for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive) { if objIncompl.Key != "" { incompObjNotFound = false break } } if !incompObjNotFound { logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err) return } function = "GetObject(bucketName, objectName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, } newReader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } newReadBytes, err := io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } newReader.Close() if !bytes.Equal(newReadBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch", err) return } function = "FGetObject(bucketName, objectName, fileName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "fileName": fileName + "-f", } err = c.FGetObject(context.Background(), bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FgetObject failed", err) return } // Generate presigned HEAD object url. function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "expires": 3600 * time.Second, } presignedHeadURL, err := c.PresignedHeadObject(context.Background(), bucketName, objectName, 3600*time.Second, nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) return } httpClient := &http.Client{ // Setting a sensible time out of 30secs to wait for response // headers. Request is pro-actively canceled after 30secs // with no response. Timeout: 30 * time.Second, Transport: createHTTPTransport(), } req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err) return } // Verify if presigned url works. resp, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err) return } if resp.StatusCode != http.StatusOK { logError(testName, function, args, startTime, "", "PresignedHeadObject URL returns status "+string(resp.StatusCode), err) return } if resp.Header.Get("ETag") == "" { logError(testName, function, args, startTime, "", "Got empty ETag", err) return } resp.Body.Close() // Generate presigned GET object url. function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "expires": 3600 * time.Second, } presignedGetURL, err := c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) return } // Verify if presigned url works. req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) return } resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } if resp.StatusCode != http.StatusOK { logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) return } newPresignedBytes, err := io.ReadAll(resp.Body) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } resp.Body.Close() if !bytes.Equal(newPresignedBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch", err) return } // Set request parameters. reqParams := make(url.Values) reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") // Generate presigned GET object url. args["reqParams"] = reqParams presignedGetURL, err = c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, reqParams) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) return } // Verify if presigned url works. req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) return } resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } if resp.StatusCode != http.StatusOK { logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) return } newPresignedBytes, err = io.ReadAll(resp.Body) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } if !bytes.Equal(newPresignedBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch", err) return } // Verify content disposition. if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { logError(testName, function, args, startTime, "", "wrong Content-Disposition received ", err) return } function = "PresignedPutObject(bucketName, objectName, expires)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName + "-presigned", "expires": 3600 * time.Second, } presignedPutURL, err := c.PresignedPutObject(context.Background(), bucketName, objectName+"-presigned", 3600*time.Second) if err != nil { logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) return } // Generate data more than 32K buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024) req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf)) if err != nil { logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err) return } resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err) return } // Download the uploaded object to verify args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName + "-presigned", } newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presigned", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject of uploaded presigned object failed", err) return } newReadBytes, err = io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed during get on presigned put object", err) return } newReader.Close() if !bytes.Equal(newReadBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch on presigned object upload verification", err) return } function = "PresignHeader(method, bucketName, objectName, expires, reqParams, extraHeaders)" functionAll += ", " + function presignExtraHeaders := map[string][]string{ "mysecret": {"abcxxx"}, } args = map[string]interface{}{ "method": "PUT", "bucketName": bucketName, "objectName": objectName + "-presign-custom", "expires": 3600 * time.Second, "extraHeaders": presignExtraHeaders, } _, err = c.PresignHeader(context.Background(), "PUT", bucketName, objectName+"-presign-custom", 3600*time.Second, nil, presignExtraHeaders) if err == nil { logError(testName, function, args, startTime, "", "Presigned with extra headers succeeded", err) return } os.Remove(fileName) os.Remove(fileName + "-f") logSuccess(testName, functionAll, args, startTime) } // Test get object with GetObject with context func testGetObjectContext() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(ctx, bucketName, objectName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx cancel() r, err := c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed unexpectedly", err) return } if _, err = r.Stat(); err == nil { logError(testName, function, args, startTime, "", "GetObject should fail on short timeout", err) return } r.Close() ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) args["ctx"] = ctx defer cancel() // Read the data back r, err = c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "object Stat call failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match: want "+string(bufSize)+", got"+string(st.Size), err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", "object Close() call failed", err) return } logSuccess(testName, function, args, startTime) } // Test get object with FGetObject with a user provided context func testFGetObjectContext() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FGetObject(ctx, bucketName, objectName, fileName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", "fileName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-1-MB"] reader := getDataReader("datafile-1-MB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx defer cancel() fileName := "tempfile-context" args["fileName"] = fileName // Read the data back err = c.FGetObject(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) if err == nil { logError(testName, function, args, startTime, "", "FGetObject should fail on short timeout", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() // Read the data back err = c.FGetObject(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FGetObject with long timeout failed", err) return } if err = os.Remove(fileName + "-fcontext"); err != nil { logError(testName, function, args, startTime, "", "Remove file failed", err) return } logSuccess(testName, function, args, startTime) } // Test get object with GetObject with a user provided context func testGetObjectRanges() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(ctx, bucketName, objectName, fileName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", "fileName": "", } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() rng := rand.NewSource(time.Now().UnixNano()) c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rng, "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() // Save the data objectName := randString(60, rng, "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back tests := []struct { start int64 end int64 }{ { start: 1024, end: 1024 + 1<<20, }, { start: 20e6, end: 20e6 + 10000, }, { start: 40e6, end: 40e6 + 10000, }, { start: 60e6, end: 60e6 + 10000, }, { start: 80e6, end: 80e6 + 10000, }, { start: 120e6, end: int64(bufSize), }, } for _, test := range tests { wantRC := getDataReader("datafile-129-MB") io.CopyN(io.Discard, wantRC, test.start) want := mustCrcReader(io.LimitReader(wantRC, test.end-test.start+1)) opts := minio.GetObjectOptions{} opts.SetRange(test.start, test.end) args["opts"] = fmt.Sprintf("%+v", test) obj, err := c.GetObject(ctx, bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "FGetObject with long timeout failed", err) return } err = crcMatches(obj, want) if err != nil { logError(testName, function, args, startTime, "", fmt.Sprintf("GetObject offset %d -> %d", test.start, test.end), err) return } } logSuccess(testName, function, args, startTime) } // Test get object ACLs with GetObjectACL with custom provided context func testGetObjectACLContext() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObjectACL(ctx, bucketName, objectName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-1-MB"] reader := getDataReader("datafile-1-MB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Add meta data to add a canned acl metaData := map[string]string{ "X-Amz-Acl": "public-read-write", } _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ ContentType: "binary/octet-stream", UserMetadata: metaData, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) args["ctx"] = ctx defer cancel() // Read the data back objectInfo, getObjectACLErr := c.GetObjectACL(ctx, bucketName, objectName) if getObjectACLErr != nil { logError(testName, function, args, startTime, "", "GetObjectACL failed. ", getObjectACLErr) return } s, ok := objectInfo.Metadata["X-Amz-Acl"] if !ok { logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Acl\"", nil) return } if len(s) != 1 { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) return } // Do a very limited testing if this is not AWS S3 if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { if s[0] != "private" { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"private\" but got"+fmt.Sprintf("%q", s[0]), nil) return } logSuccess(testName, function, args, startTime) return } if s[0] != "public-read-write" { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil) return } bufSize = dataFileMap["datafile-1-MB"] reader2 := getDataReader("datafile-1-MB") defer reader2.Close() // Save the data objectName = randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Add meta data to add a canned acl metaData = map[string]string{ "X-Amz-Grant-Read": "id=fooread@minio.go", "X-Amz-Grant-Write": "id=foowrite@minio.go", } _, err = c.PutObject(context.Background(), bucketName, objectName, reader2, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) args["ctx"] = ctx defer cancel() // Read the data back objectInfo, getObjectACLErr = c.GetObjectACL(ctx, bucketName, objectName) if getObjectACLErr == nil { logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr) return } if len(objectInfo.Metadata) != 3 { logError(testName, function, args, startTime, "", "GetObjectACL fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil) return } s, ok = objectInfo.Metadata["X-Amz-Grant-Read"] if !ok { logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Read\"", nil) return } if len(s) != 1 { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) return } if s[0] != "fooread@minio.go" { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil) return } s, ok = objectInfo.Metadata["X-Amz-Grant-Write"] if !ok { logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Write\"", nil) return } if len(s) != 1 { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) return } if s[0] != "foowrite@minio.go" { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil) return } logSuccess(testName, function, args, startTime) } // Test validates putObject with context to see if request cancellation is honored for V2. func testPutObjectContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(ctx, bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", "size": "", "opts": "", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datatfile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) args["objectName"] = objectName ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) args["ctx"] = ctx args["size"] = bufSize defer cancel() _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject with short timeout failed", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) args["ctx"] = ctx defer cancel() reader = getDataReader("datafile-33-kB") defer reader.Close() _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject with long timeout failed", err) return } logSuccess(testName, function, args, startTime) } // Test get object with GetObject with custom context func testGetObjectContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(ctx, bucketName, objectName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx cancel() r, err := c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed unexpectedly", err) return } if _, err = r.Stat(); err == nil { logError(testName, function, args, startTime, "", "GetObject should fail on short timeout", err) return } r.Close() ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() // Read the data back r, err = c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject shouldn't fail on longer timeout", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "object Stat call failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(bufSize)+" got "+string(st.Size), err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", " object Close() call failed", err) return } logSuccess(testName, function, args, startTime) } // Test get object with FGetObject with custom context func testFGetObjectContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FGetObject(ctx, bucketName, objectName,fileName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", "fileName": "", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket call failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datatfile-1-MB"] reader := getDataReader("datafile-1-MB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx defer cancel() fileName := "tempfile-context" args["fileName"] = fileName // Read the data back err = c.FGetObject(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) if err == nil { logError(testName, function, args, startTime, "", "FGetObject should fail on short timeout", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() // Read the data back err = c.FGetObject(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FGetObject call shouldn't fail on long timeout", err) return } if err = os.Remove(fileName + "-fcontext"); err != nil { logError(testName, function, args, startTime, "", "Remove file failed", err) return } logSuccess(testName, function, args, startTime) } // Test list object v1 and V2 func testListObjects() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ListObjects(bucketName, objectPrefix, recursive, doneCh)" args := map[string]interface{}{ "bucketName": "", "objectPrefix": "", "recursive": "true", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) testObjects := []struct { name string storageClass string }{ // Special characters {"foo bar", "STANDARD"}, {"foo-%", "STANDARD"}, {"random-object-1", "STANDARD"}, {"random-object-2", "REDUCED_REDUNDANCY"}, } for i, object := range testObjects { bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() _, err = c.PutObject(context.Background(), bucketName, object.name, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: object.storageClass}) if err != nil { logError(testName, function, args, startTime, "", fmt.Sprintf("PutObject %d call failed", i+1), err) return } } testList := func(listFn func(context.Context, string, minio.ListObjectsOptions) <-chan minio.ObjectInfo, bucket string, opts minio.ListObjectsOptions) { var objCursor int // check for object name and storage-class from listing object result for objInfo := range listFn(context.Background(), bucket, opts) { if objInfo.Err != nil { logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err) return } if objInfo.Key != testObjects[objCursor].name { logError(testName, function, args, startTime, "", "ListObjects does not return expected object name", err) return } if objInfo.StorageClass != testObjects[objCursor].storageClass { // Ignored as Gateways (Azure/GCS etc) wont return storage class logIgnored(testName, function, args, startTime, "ListObjects doesn't return expected storage class") } objCursor++ } if objCursor != len(testObjects) { logError(testName, function, args, startTime, "", "ListObjects returned unexpected number of items", errors.New("")) return } } testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true, UseV1: true}) testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true}) testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true, WithMetadata: true}) logSuccess(testName, function, args, startTime) } // testCors is runnable against S3 itself. // Just provide the env var MINIO_GO_TEST_BUCKET_CORS with bucket that is public and WILL BE DELETED. // Recreate this manually each time. Minio-go SDK does not support calling // SetPublicBucket (put-public-access-block) on S3, otherwise we could script the whole thing. func testCors() { ctx := context.Background() startTime := time.Now() testName := getFuncName() function := "SetBucketCors(bucketName, cors)" args := map[string]interface{}{ "bucketName": "", "cors": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Create or reuse a bucket that will get cors settings applied to it and deleted when done bucketName := os.Getenv("MINIO_GO_TEST_BUCKET_CORS") if bucketName == "" { bucketName = randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") err = c.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } } args["bucketName"] = bucketName defer cleanupBucket(bucketName, c) publicPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:*"],"Resource":["arn:aws:s3:::` + bucketName + `", "arn:aws:s3:::` + bucketName + `/*"]}]}` err = c.SetBucketPolicy(ctx, bucketName, publicPolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // Upload an object for testing. objectContents := `some-text-file-contents` reader := strings.NewReader(objectContents) bufSize := int64(len(objectContents)) objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } bucketURL := c.EndpointURL().String() + "/" + bucketName + "/" objectURL := bucketURL + objectName httpClient := &http.Client{ Timeout: 30 * time.Second, Transport: createHTTPTransport(), } errStrAccessForbidden := `AccessForbiddenCORSResponse: This CORS request is not allowed. This is usually because the evalution of Origin, request method / Access-Control-Request-Method or Access-Control-Request-Headers are not whitelisted` testCases := []struct { name string // Cors rules to apply applyCorsRules []cors.Rule // Outbound request info method string url string headers map[string]string // Wanted response wantStatus int wantHeaders map[string]string wantBodyContains string }{ { name: "apply bucket rules", applyCorsRules: []cors.Rule{ { AllowedOrigin: []string{"https"}, // S3 documents 'https' origin, but it does not actually work, see test below. AllowedMethod: []string{"PUT"}, AllowedHeader: []string{"*"}, }, { AllowedOrigin: []string{"http://www.example1.com"}, AllowedMethod: []string{"PUT"}, AllowedHeader: []string{"*"}, ExposeHeader: []string{"x-amz-server-side-encryption", "x-amz-request-id"}, MaxAgeSeconds: 3600, }, { AllowedOrigin: []string{"http://www.example2.com"}, AllowedMethod: []string{"POST"}, AllowedHeader: []string{"X-My-Special-Header"}, ExposeHeader: []string{"X-AMZ-Request-ID"}, }, { AllowedOrigin: []string{"http://www.example3.com"}, AllowedMethod: []string{"PUT"}, AllowedHeader: []string{"X-Example-3-Special-Header"}, MaxAgeSeconds: 10, }, { AllowedOrigin: []string{"*"}, AllowedMethod: []string{"GET"}, AllowedHeader: []string{"*"}, ExposeHeader: []string{"x-amz-request-id", "X-AMZ-server-side-encryption"}, MaxAgeSeconds: 3600, }, { AllowedOrigin: []string{"http://multiplemethodstest.com"}, AllowedMethod: []string{"POST", "PUT", "DELETE"}, AllowedHeader: []string{"x-abc-*", "x-def-*"}, }, { AllowedOrigin: []string{"http://UPPERCASEEXAMPLE.com"}, AllowedMethod: []string{"DELETE"}, }, { AllowedOrigin: []string{"https://*"}, AllowedMethod: []string{"DELETE"}, AllowedHeader: []string{"x-abc-*", "x-def-*"}, }, }, }, { name: "preflight to object url matches example1 rule", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Method": "PUT", "Access-Control-Request-Headers": "x-another-header,x-could-be-anything", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Origin": "http://www.example1.com", "Access-Control-Allow-Methods": "PUT", "Access-Control-Allow-Headers": "x-another-header,x-could-be-anything", "Access-Control-Allow-Credentials": "true", "Access-Control-Max-Age": "3600", "Content-Length": "0", // S3 additionally sets the following headers here, MinIO follows fetch spec and does not: // "Access-Control-Expose-Headers": "", }, }, { name: "preflight to bucket url matches example1 rule", method: http.MethodOptions, url: bucketURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Method": "PUT", "Access-Control-Request-Headers": "x-another-header,x-could-be-anything", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Origin": "http://www.example1.com", "Access-Control-Allow-Methods": "PUT", "Access-Control-Allow-Headers": "x-another-header,x-could-be-anything", "Access-Control-Allow-Credentials": "true", "Access-Control-Max-Age": "3600", "Content-Length": "0", }, }, { name: "preflight matches example2 rule with header given", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.example2.com", "Access-Control-Request-Method": "POST", "Access-Control-Request-Headers": "X-My-Special-Header", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Origin": "http://www.example2.com", "Access-Control-Allow-Methods": "POST", "Access-Control-Allow-Headers": "x-my-special-header", "Access-Control-Allow-Credentials": "true", "Access-Control-Max-Age": "", "Content-Length": "0", }, }, { name: "preflight matches example2 rule with no header given", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.example2.com", "Access-Control-Request-Method": "POST", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Origin": "http://www.example2.com", "Access-Control-Allow-Methods": "POST", "Access-Control-Allow-Headers": "", "Access-Control-Allow-Credentials": "true", "Access-Control-Max-Age": "", "Content-Length": "0", }, }, { name: "preflight matches wildcard origin rule", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.couldbeanything.com", "Access-Control-Request-Method": "GET", "Access-Control-Request-Headers": "x-custom-header,x-other-custom-header", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Methods": "GET", "Access-Control-Allow-Headers": "x-custom-header,x-other-custom-header", "Access-Control-Allow-Credentials": "", "Access-Control-Max-Age": "3600", "Content-Length": "0", }, }, { name: "preflight does not match any rule", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.couldbeanything.com", "Access-Control-Request-Method": "DELETE", }, wantStatus: http.StatusForbidden, wantBodyContains: errStrAccessForbidden, }, { name: "preflight does not match example1 rule because of method", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Method": "POST", }, wantStatus: http.StatusForbidden, wantBodyContains: errStrAccessForbidden, }, { name: "s3 processes cors rules even when request is not preflight if cors headers present test get", method: http.MethodGet, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Headers": "x-another-header,x-could-be-anything", "Access-Control-Request-Method": "PUT", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Origin": "http://www.example1.com", "Access-Control-Expose-Headers": "x-amz-server-side-encryption,x-amz-request-id", // S3 additionally sets the following headers here, MinIO follows fetch spec and does not: // "Access-Control-Allow-Headers": "x-another-header,x-could-be-anything", // "Access-Control-Allow-Methods": "PUT", // "Access-Control-Max-Age": "3600", }, }, { name: "s3 processes cors rules even when request is not preflight if cors headers present test put", method: http.MethodPut, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Method": "GET", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "", "Access-Control-Allow-Origin": "*", "Access-Control-Expose-Headers": "x-amz-request-id,x-amz-server-side-encryption", // S3 additionally sets the following headers here, MinIO follows fetch spec and does not: // "Access-Control-Allow-Headers": "x-another-header,x-could-be-anything", // "Access-Control-Allow-Methods": "PUT", // "Access-Control-Max-Age": "3600", }, }, { name: "s3 processes cors rules even when request is not preflight but there is no rule match", method: http.MethodGet, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Headers": "x-another-header,x-could-be-anything", "Access-Control-Request-Method": "DELETE", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Methods": "", "Access-Control-Allow-Origin": "", "Access-Control-Allow-Headers": "", "Access-Control-Allow-Credentials": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "get request matches wildcard origin rule and returns cors headers", method: http.MethodGet, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "", "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "x-amz-request-id,X-AMZ-server-side-encryption", // S3 returns the following headers, MinIO follows fetch spec and does not: // "Access-Control-Max-Age": "3600", // "Access-Control-Allow-Methods": "GET", }, }, { name: "head request does not match rule and returns no cors headers", method: http.MethodHead, url: objectURL, headers: map[string]string{ "Origin": "http://www.nomatchingdomainfound.com", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "", "Access-Control-Allow-Methods": "", "Access-Control-Allow-Origin": "", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "put request with origin does not match rule and returns no cors headers", method: http.MethodPut, url: objectURL, headers: map[string]string{ "Origin": "http://www.nomatchingdomainfound.com", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "", "Access-Control-Allow-Methods": "", "Access-Control-Allow-Origin": "", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "put request with no origin does not match rule and returns no cors headers", method: http.MethodPut, url: objectURL, headers: map[string]string{}, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "", "Access-Control-Allow-Methods": "", "Access-Control-Allow-Origin": "", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "preflight for delete request with wildcard origin does not match", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.notsecureexample.com", "Access-Control-Request-Method": "DELETE", }, wantStatus: http.StatusForbidden, wantBodyContains: errStrAccessForbidden, }, { name: "preflight for delete request with wildcard https origin matches secureexample", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "https://www.secureexample.com", "Access-Control-Request-Method": "DELETE", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Methods": "DELETE", "Access-Control-Allow-Origin": "https://www.secureexample.com", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "preflight for delete request matches secureexample with wildcard https origin and request headers", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "https://www.secureexample.com", "Access-Control-Request-Method": "DELETE", "Access-Control-Request-Headers": "x-abc-1,x-abc-second,x-def-1", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Methods": "DELETE", "Access-Control-Allow-Origin": "https://www.secureexample.com", "Access-Control-Allow-Headers": "x-abc-1,x-abc-second,x-def-1", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "preflight for delete request matches secureexample rejected because request header does not match", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "https://www.secureexample.com", "Access-Control-Request-Method": "DELETE", "Access-Control-Request-Headers": "x-abc-1,x-abc-second,x-def-1,x-does-not-match", }, wantStatus: http.StatusForbidden, wantBodyContains: errStrAccessForbidden, }, { name: "preflight with https origin is documented by s3 as matching but it does not match", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "https://www.securebutdoesnotmatch.com", "Access-Control-Request-Method": "PUT", }, wantStatus: http.StatusForbidden, wantBodyContains: errStrAccessForbidden, }, { name: "put no origin no match returns no cors headers", method: http.MethodPut, url: objectURL, headers: map[string]string{}, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "", "Access-Control-Allow-Methods": "", "Access-Control-Allow-Origin": "", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "put with origin match example1 returns cors headers", method: http.MethodPut, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Origin": "http://www.example1.com", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "x-amz-server-side-encryption,x-amz-request-id", // S3 returns the following headers, MinIO follows fetch spec and does not: // "Access-Control-Max-Age": "3600", // "Access-Control-Allow-Methods": "PUT", }, }, { name: "put with origin and header match example1 returns cors headers", method: http.MethodPut, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "x-could-be-anything": "myvalue", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Origin": "http://www.example1.com", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "x-amz-server-side-encryption,x-amz-request-id", // S3 returns the following headers, MinIO follows fetch spec and does not: // "Access-Control-Max-Age": "3600", // "Access-Control-Allow-Methods": "PUT", }, }, { name: "put no match found returns no cors headers", method: http.MethodPut, url: objectURL, headers: map[string]string{ "Origin": "http://www.unmatchingdomain.com", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "", "Access-Control-Allow-Methods": "", "Access-Control-Allow-Origin": "", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "put with origin match example3 returns cors headers", method: http.MethodPut, url: objectURL, headers: map[string]string{ "Origin": "http://www.example3.com", "X-My-Special-Header": "myvalue", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Origin": "http://www.example3.com", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", // S3 returns the following headers, MinIO follows fetch spec and does not: // "Access-Control-Max-Age": "10", // "Access-Control-Allow-Methods": "PUT", }, }, { name: "preflight matches example1 rule headers case is incorrect", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Method": "PUT", // Fetch standard guarantees that these are sent lowercase, here we test what happens when they are not. "Access-Control-Request-Headers": "X-Another-Header,X-Could-Be-Anything", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Origin": "http://www.example1.com", "Access-Control-Allow-Methods": "PUT", "Access-Control-Allow-Headers": "x-another-header,x-could-be-anything", "Access-Control-Allow-Credentials": "true", "Access-Control-Max-Age": "3600", "Content-Length": "0", // S3 returns the following headers, MinIO follows fetch spec and does not: // "Access-Control-Expose-Headers": "x-amz-server-side-encryption,x-amz-request-id", }, }, { name: "preflight matches example1 rule headers are not sorted", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Method": "PUT", // Fetch standard guarantees that these are sorted, test what happens when they are not. "Access-Control-Request-Headers": "a-customer-header,b-should-be-last", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Origin": "http://www.example1.com", "Access-Control-Allow-Methods": "PUT", "Access-Control-Allow-Headers": "a-customer-header,b-should-be-last", "Access-Control-Allow-Credentials": "true", "Access-Control-Max-Age": "3600", "Content-Length": "0", // S3 returns the following headers, MinIO follows fetch spec and does not: // "Access-Control-Expose-Headers": "x-amz-server-side-encryption,x-amz-request-id", }, }, { name: "preflight with case sensitivity in origin matches uppercase", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://UPPERCASEEXAMPLE.com", "Access-Control-Request-Method": "DELETE", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Methods": "DELETE", "Access-Control-Allow-Origin": "http://UPPERCASEEXAMPLE.com", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "preflight with case sensitivity in origin does not match when lowercase", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://uppercaseexample.com", "Access-Control-Request-Method": "DELETE", }, wantStatus: http.StatusForbidden, wantBodyContains: errStrAccessForbidden, }, { name: "preflight match upper case with unknown header but no header restrictions", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://UPPERCASEEXAMPLE.com", "Access-Control-Request-Method": "DELETE", "Access-Control-Request-Headers": "x-unknown-1", }, wantStatus: http.StatusForbidden, wantBodyContains: errStrAccessForbidden, }, { name: "preflight for delete request matches multiplemethodstest.com origin and request headers", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://multiplemethodstest.com", "Access-Control-Request-Method": "DELETE", "Access-Control-Request-Headers": "x-abc-1", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Origin": "http://multiplemethodstest.com", "Access-Control-Allow-Headers": "x-abc-1", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", // S3 returns POST, PUT, DELETE here, MinIO does not as spec does not require it. // "Access-Control-Allow-Methods": "DELETE", }, }, { name: "delete request goes ahead because cors is only for browsers and does not block on the server side", method: http.MethodDelete, url: objectURL, headers: map[string]string{ "Origin": "http://www.justrandom.com", }, wantStatus: http.StatusNoContent, }, } for i, test := range testCases { testName := fmt.Sprintf("%s_%d_%s", testName, i+1, strings.ReplaceAll(test.name, " ", "_")) // Apply the CORS rules if test.applyCorsRules != nil { corsConfig := &cors.Config{ CORSRules: test.applyCorsRules, } err = c.SetBucketCors(ctx, bucketName, corsConfig) if err != nil { logError(testName, function, args, startTime, "", "SetBucketCors failed to apply", err) return } } // Make request if test.method != "" && test.url != "" { req, err := http.NewRequestWithContext(ctx, test.method, test.url, nil) if err != nil { logError(testName, function, args, startTime, "", "HTTP request creation failed", err) return } req.Header.Set("User-Agent", "MinIO-go-FunctionalTest/"+appVersion) for k, v := range test.headers { req.Header.Set(k, v) } resp, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "HTTP request failed", err) return } defer resp.Body.Close() // Check returned status code if resp.StatusCode != test.wantStatus { errStr := fmt.Sprintf(" incorrect status code in response, want: %d, got: %d", test.wantStatus, resp.StatusCode) logError(testName, function, args, startTime, "", errStr, nil) return } // Check returned body if test.wantBodyContains != "" { body, err := io.ReadAll(resp.Body) if err != nil { logError(testName, function, args, startTime, "", "Failed to read response body", err) return } if !strings.Contains(string(body), test.wantBodyContains) { errStr := fmt.Sprintf(" incorrect body in response, want: %s, in got: %s", test.wantBodyContains, string(body)) logError(testName, function, args, startTime, "", errStr, nil) return } } // Check returned response headers for k, v := range test.wantHeaders { gotVal := resp.Header.Get(k) if k == "Access-Control-Expose-Headers" { // MinIO returns this in canonical form, S3 does not. gotVal = strings.ToLower(gotVal) v = strings.ToLower(v) } // Remove all spaces, S3 adds spaces after CSV values in headers, MinIO does not. gotVal = strings.ReplaceAll(gotVal, " ", "") if gotVal != v { errStr := fmt.Sprintf(" incorrect header in response, want: %s: '%s', got: '%s'", k, v, gotVal) logError(testName, function, args, startTime, "", errStr, nil) return } } } logSuccess(testName, function, args, startTime) } logSuccess(testName, function, args, startTime) } func testCorsSetGetDelete() { ctx := context.Background() startTime := time.Now() testName := getFuncName() function := "SetBucketCors(bucketName, cors)" args := map[string]interface{}{ "bucketName": "", "cors": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Set the CORS rules on the new bucket corsRules := []cors.Rule{ { AllowedOrigin: []string{"http://www.example1.com"}, AllowedMethod: []string{"PUT"}, AllowedHeader: []string{"*"}, }, { AllowedOrigin: []string{"http://www.example2.com"}, AllowedMethod: []string{"POST"}, AllowedHeader: []string{"X-My-Special-Header"}, }, { AllowedOrigin: []string{"*"}, AllowedMethod: []string{"GET"}, AllowedHeader: []string{"*"}, }, } corsConfig := cors.NewConfig(corsRules) err = c.SetBucketCors(ctx, bucketName, corsConfig) if err != nil { logError(testName, function, args, startTime, "", "SetBucketCors failed to apply", err) return } // Get the rules and check they match what we set gotCorsConfig, err := c.GetBucketCors(ctx, bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketCors failed", err) return } if !reflect.DeepEqual(corsConfig, gotCorsConfig) { msg := fmt.Sprintf("GetBucketCors returned unexpected rules, expected: %+v, got: %+v", corsConfig, gotCorsConfig) logError(testName, function, args, startTime, "", msg, nil) return } // Delete the rules err = c.SetBucketCors(ctx, bucketName, nil) if err != nil { logError(testName, function, args, startTime, "", "SetBucketCors failed to delete", err) return } // Get the rules and check they are now empty gotCorsConfig, err = c.GetBucketCors(ctx, bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketCors failed", err) return } if gotCorsConfig != nil { logError(testName, function, args, startTime, "", "GetBucketCors returned unexpected rules", nil) return } logSuccess(testName, function, args, startTime) } // Test deleting multiple objects with object retention set in Governance mode func testRemoveObjects() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "RemoveObjects(bucketName, objectsCh, opts)" args := map[string]interface{}{ "bucketName": "", "objectPrefix": "", "recursive": "true", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Error uploading object", err) return } // Replace with smaller... bufSize = dataFileMap["datafile-10-kB"] reader = getDataReader("datafile-10-kB") defer reader.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Error uploading object", err) } t := time.Date(2030, time.April, 25, 14, 0, 0, 0, time.UTC) m := minio.RetentionMode(minio.Governance) opts := minio.PutObjectRetentionOptions{ GovernanceBypass: false, RetainUntilDate: &t, Mode: &m, } err = c.PutObjectRetention(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "Error setting retention", err) return } objectsCh := make(chan minio.ObjectInfo) // Send object names that are needed to be removed to objectsCh go func() { defer close(objectsCh) // List all objects from a bucket-name with a matching prefix. for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Recursive: true}) { if object.Err != nil { logError(testName, function, args, startTime, "", "Error listing objects", object.Err) return } objectsCh <- object } }() for rErr := range c.RemoveObjects(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{}) { // Error is expected here because Retention is set on the object // and RemoveObjects is called without Bypass Governance if rErr.Err == nil { logError(testName, function, args, startTime, "", "Expected error during deletion", nil) return } } objectsCh1 := make(chan minio.ObjectInfo) // Send object names that are needed to be removed to objectsCh go func() { defer close(objectsCh1) // List all objects from a bucket-name with a matching prefix. for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Recursive: true}) { if object.Err != nil { logError(testName, function, args, startTime, "", "Error listing objects", object.Err) return } objectsCh1 <- object } }() opts1 := minio.RemoveObjectsOptions{ GovernanceBypass: true, } for rErr := range c.RemoveObjects(context.Background(), bucketName, objectsCh1, opts1) { // Error is not expected here because Retention is set on the object // and RemoveObjects is called with Bypass Governance logError(testName, function, args, startTime, "", "Error detected during deletion", rErr.Err) return } // Delete all objects and buckets if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Test deleting multiple objects with object retention set in Governance mode, via iterators func testRemoveObjectsIter() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "RemoveObjects(bucketName, objectsCh, opts)" args := map[string]interface{}{ "bucketName": "", "objectPrefix": "", "recursive": "true", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Error uploading object", err) return } // Replace with smaller... bufSize = dataFileMap["datafile-10-kB"] reader = getDataReader("datafile-10-kB") defer reader.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Error uploading object", err) } t := time.Date(2030, time.April, 25, 14, 0, 0, 0, time.UTC) m := minio.RetentionMode(minio.Governance) opts := minio.PutObjectRetentionOptions{ GovernanceBypass: false, RetainUntilDate: &t, Mode: &m, } err = c.PutObjectRetention(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "Error setting retention", err) return } objectsIter := c.ListObjectsIter(context.Background(), bucketName, minio.ListObjectsOptions{ WithVersions: true, Recursive: true, }) results, err := c.RemoveObjectsWithIter(context.Background(), bucketName, objectsIter, minio.RemoveObjectsOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Error sending delete request", err) return } for result := range results { if result.Err != nil { // Error is expected here because Retention is set on the object // and RemoveObjects is called without Bypass Governance break } logError(testName, function, args, startTime, "", "Expected error during deletion", nil) return } objectsIter = c.ListObjectsIter(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Recursive: true}) results, err = c.RemoveObjectsWithIter(context.Background(), bucketName, objectsIter, minio.RemoveObjectsOptions{ GovernanceBypass: true, }) if err != nil { logError(testName, function, args, startTime, "", "Error sending delete request", err) return } for result := range results { if result.Err != nil { // Error is not expected here because Retention is set on the object // and RemoveObjects is called with Bypass Governance logError(testName, function, args, startTime, "", "Error detected during deletion", result.Err) return } } // Delete all objects and buckets if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Test get bucket tags func testGetBucketTagging() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetBucketTagging(bucketName)" args := map[string]interface{}{ "bucketName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } _, err = c.GetBucketTagging(context.Background(), bucketName) if minio.ToErrorResponse(err).Code != minio.NoSuchTagSet { logError(testName, function, args, startTime, "", "Invalid error from server failed", err) return } if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Test setting tags for bucket func testSetBucketTagging() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "SetBucketTagging(bucketName, tags)" args := map[string]interface{}{ "bucketName": "", "tags": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } _, err = c.GetBucketTagging(context.Background(), bucketName) if minio.ToErrorResponse(err).Code != minio.NoSuchTagSet { logError(testName, function, args, startTime, "", "Invalid error from server", err) return } tag := randString(60, rand.NewSource(time.Now().UnixNano()), "") expectedValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") t, err := tags.MapToBucketTags(map[string]string{ tag: expectedValue, }) args["tags"] = t.String() if err != nil { logError(testName, function, args, startTime, "", "tags.MapToBucketTags failed", err) return } err = c.SetBucketTagging(context.Background(), bucketName, t) if err != nil { logError(testName, function, args, startTime, "", "SetBucketTagging failed", err) return } tagging, err := c.GetBucketTagging(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketTagging failed", err) return } if tagging.ToMap()[tag] != expectedValue { msg := fmt.Sprintf("Tag %s; got value %s; wanted %s", tag, tagging.ToMap()[tag], expectedValue) logError(testName, function, args, startTime, "", msg, err) return } // Delete all objects and buckets if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Test removing bucket tags func testRemoveBucketTagging() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "RemoveBucketTagging(bucketName)" args := map[string]interface{}{ "bucketName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } _, err = c.GetBucketTagging(context.Background(), bucketName) if minio.ToErrorResponse(err).Code != minio.NoSuchTagSet { logError(testName, function, args, startTime, "", "Invalid error from server", err) return } tag := randString(60, rand.NewSource(time.Now().UnixNano()), "") expectedValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") t, err := tags.MapToBucketTags(map[string]string{ tag: expectedValue, }) if err != nil { logError(testName, function, args, startTime, "", "tags.MapToBucketTags failed", err) return } err = c.SetBucketTagging(context.Background(), bucketName, t) if err != nil { logError(testName, function, args, startTime, "", "SetBucketTagging failed", err) return } tagging, err := c.GetBucketTagging(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketTagging failed", err) return } if tagging.ToMap()[tag] != expectedValue { msg := fmt.Sprintf("Tag %s; got value %s; wanted %s", tag, tagging.ToMap()[tag], expectedValue) logError(testName, function, args, startTime, "", msg, err) return } err = c.RemoveBucketTagging(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "RemoveBucketTagging failed", err) return } _, err = c.GetBucketTagging(context.Background(), bucketName) if minio.ToErrorResponse(err).Code != minio.NoSuchTagSet { logError(testName, function, args, startTime, "", "Invalid error from server", err) return } // Delete all objects and buckets if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Convert string to bool and always return false if any error func mustParseBool(str string) bool { b, err := strconv.ParseBool(str) if err != nil { return false } return b } // wantChecksums is a map of expected checksums for an object. type wantChecksums map[minio.ChecksumType]string // cmpChecksum compares the checksums of an object against expected values. func cmpChecksum(oi minio.ObjectInfo, chksums wantChecksums) error { if oi.ChecksumCRC64NVME != chksums[minio.ChecksumCRC64NVME] { return fmt.Errorf("Checksum mismatch for CRC64NVME, want: %s, got: %s", chksums[minio.ChecksumCRC64NVME], oi.ChecksumCRC64NVME) } if oi.ChecksumCRC32C != chksums[minio.ChecksumCRC32C] { return fmt.Errorf("Checksum mismatch for CRC32C, want: %s, got: %s", chksums[minio.ChecksumCRC32C], oi.ChecksumCRC32C) } if oi.ChecksumCRC32 != chksums[minio.ChecksumCRC32] { return fmt.Errorf("Checksum mismatch for CRC32, want: %s, got: %s", chksums[minio.ChecksumCRC32], oi.ChecksumCRC32) } if oi.ChecksumSHA1 != chksums[minio.ChecksumSHA1] { return fmt.Errorf("Checksum mismatch for SHA1, want: %s, got: %s", chksums[minio.ChecksumSHA1], oi.ChecksumSHA1) } if oi.ChecksumSHA256 != chksums[minio.ChecksumSHA256] { return fmt.Errorf("Checksum mismatch for SHA256, want: %s, got: %s", chksums[minio.ChecksumSHA256], oi.ChecksumSHA256) } return nil } func main() { slog.SetDefault(slog.New(slog.NewJSONHandler( os.Stdout, &slog.HandlerOptions{ Level: slog.LevelInfo, ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { if a.Key == slog.MessageKey || a.Value.String() == "" { return slog.Attr{} } return a }, }, ))) tls := mustParseBool(os.Getenv(enableHTTPS)) kms := mustParseBool(os.Getenv(enableKMS)) if os.Getenv(enableKMS) == "" { // Default to KMS tests. kms = true } // execute tests if isFullMode() { testCopyObjectWithChecksums() testReplaceObjectWithChecksums() testCorsSetGetDelete() testCors() testListMultipartUpload() testGetObjectAttributes() testGetObjectAttributesErrorCases() testMakeBucketErrorV2() testGetObjectClosedTwiceV2() testFPutObjectV2() testMakeBucketRegionsV2() testGetObjectReadSeekFunctionalV2() testGetObjectReadAtFunctionalV2() testGetObjectRanges() testCopyObjectV2() testFunctionalV2() testComposeObjectErrorCasesV2() testCompose10KSourcesV2() testUserMetadataCopyingV2() testPutObjectWithChecksums() testPutObjectWithTrailingChecksums() testPutMultipartObjectWithChecksums() testPutObject0ByteV2() testPutObjectMetadataNonUSASCIIV2() testPutObjectNoLengthV2() testPutObjectsUnknownV2() testGetObjectContextV2() testFPutObjectContextV2() testFGetObjectContextV2() testPutObjectContextV2() testPutObjectWithVersioning() testMakeBucketError() testMakeBucketRegions() testPutObjectWithMetadata() testPutObjectReadAt() testPutObjectStreaming() testPutObjectPreconditionOnNonExistent() testGetObjectSeekEnd() testGetObjectClosedTwice() testGetObjectS3Zip() testRemoveMultipleObjects() testRemoveMultipleObjectsWithResult() testRemoveMultipleObjectsIter() testFPutObjectMultipart() testFPutObject() testGetObjectReadSeekFunctional() testGetObjectReadAtFunctional() testGetObjectReadAtWhenEOFWasReached() testPresignedPostPolicy() testPresignedPostPolicyWrongFile() testPresignedPostPolicyEmptyFileName() testCopyObject() testComposeObjectErrorCases() testCompose10KSources() testUserMetadataCopying() testBucketNotification() testFunctional() testGetObjectModified() testPutObjectUploadSeekedObject() testGetObjectContext() testFPutObjectContext() testFGetObjectContext() testGetObjectACLContext() testPutObjectContext() testStorageClassMetadataPutObject() testStorageClassInvalidMetadataPutObject() testStorageClassMetadataCopyObject() testPutObjectWithContentLanguage() testListObjects() testRemoveObjects() testRemoveObjectsIter() testListObjectVersions() testStatObjectWithVersioning() testGetObjectWithVersioning() testCopyObjectWithVersioning() testConcurrentCopyObjectWithVersioning() testComposeObjectWithVersioning() testRemoveObjectWithVersioning() testRemoveObjectsWithVersioning() testObjectTaggingWithVersioning() testTrailingChecksums() testPutObjectWithAutomaticChecksums() testGetBucketTagging() testSetBucketTagging() testRemoveBucketTagging() // SSE-C tests will only work over TLS connection. if tls { testGetObjectAttributesSSECEncryption() testSSECEncryptionPutGet() testSSECEncryptionFPut() testSSECEncryptedGetObjectReadAtFunctional() testSSECEncryptedGetObjectReadSeekFunctional() testEncryptedCopyObjectV2() testEncryptedSSECToSSECCopyObject() testEncryptedSSECToUnencryptedCopyObject() testUnencryptedToSSECCopyObject() testUnencryptedToUnencryptedCopyObject() testEncryptedEmptyObject() testDecryptedCopyObject() testSSECEncryptedToSSECCopyObjectPart() testSSECMultipartEncryptedToSSECCopyObjectPart() testSSECEncryptedToUnencryptedCopyPart() testUnencryptedToSSECCopyObjectPart() testUnencryptedToUnencryptedCopyPart() testEncryptedSSECToSSES3CopyObject() testEncryptedSSES3ToSSECCopyObject() testSSECEncryptedToSSES3CopyObjectPart() testSSES3EncryptedToSSECCopyObjectPart() } // KMS tests if kms { testSSES3EncryptionPutGet() testSSES3EncryptionFPut() testSSES3EncryptedGetObjectReadAtFunctional() testSSES3EncryptedGetObjectReadSeekFunctional() testEncryptedSSES3ToSSES3CopyObject() testEncryptedSSES3ToUnencryptedCopyObject() testUnencryptedToSSES3CopyObject() testUnencryptedToSSES3CopyObjectPart() testSSES3EncryptedToUnencryptedCopyPart() testSSES3EncryptedToSSES3CopyObjectPart() } } else { testFunctional() testFunctionalV2() } } minio-go-7.0.97/get-options_test.go000066400000000000000000000061421510244170000171670ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "fmt" "testing" ) func TestSetHeader(t *testing.T) { testCases := []struct { start int64 end int64 errVal error expected string }{ {0, 10, nil, "bytes=0-10"}, {1, 10, nil, "bytes=1-10"}, {5, 0, nil, "bytes=5-"}, {0, -5, nil, "bytes=-5"}, {0, 0, nil, "bytes=0-0"}, { 11, 10, fmt.Errorf("Invalid range specified: start=11 end=10"), "", }, {-1, 10, fmt.Errorf("Invalid range specified: start=-1 end=10"), ""}, {-1, 0, fmt.Errorf("Invalid range specified: start=-1 end=0"), ""}, {1, -5, fmt.Errorf("Invalid range specified: start=1 end=-5"), ""}, } for i, testCase := range testCases { opts := GetObjectOptions{} err := opts.SetRange(testCase.start, testCase.end) if err == nil && testCase.errVal != nil { t.Errorf("Test %d: Expected to fail with '%v' but it passed", i+1, testCase.errVal) } else if err != nil && testCase.errVal.Error() != err.Error() { t.Errorf("Test %d: Expected error '%v' but got error '%v'", i+1, testCase.errVal, err) } else if err == nil && opts.headers["Range"] != testCase.expected { t.Errorf("Test %d: Expected range header '%s', but got '%s'", i+1, testCase.expected, opts.headers["Range"]) } } } func TestCustomQueryParameters(t *testing.T) { var ( paramKey = "x-test-param" paramValue = "test-value" invalidParamKey = "invalid-test-param" invalidParamValue = "invalid-test-param" ) testCases := []struct { setParamsFunc func(o *GetObjectOptions) }{ {func(o *GetObjectOptions) { o.AddReqParam(paramKey, paramValue) o.AddReqParam(invalidParamKey, invalidParamValue) }}, {func(o *GetObjectOptions) { o.SetReqParam(paramKey, paramValue) o.SetReqParam(invalidParamKey, invalidParamValue) }}, } for i, testCase := range testCases { opts := GetObjectOptions{} testCase.setParamsFunc(&opts) // This and the following checks indirectly ensure that only the expected // valid header is added. if len(opts.reqParams) != 1 { t.Errorf("Test %d: Expected 1 kv-pair in query parameters, got %v", i+1, len(opts.reqParams)) } if v, ok := opts.reqParams[paramKey]; !ok { t.Errorf("Test %d: Expected query parameter with key %s missing", i+1, paramKey) } else if len(v) != 1 { t.Errorf("Test %d: Expected 1 value for query parameter with key %s, got %d values", i+1, paramKey, len(v)) } else if v[0] != paramValue { t.Errorf("Test %d: Expected query value %s for key %s, got %s", i+1, paramValue, paramKey, v[0]) } } } minio-go-7.0.97/go.mod000066400000000000000000000012671510244170000144420ustar00rootroot00000000000000module github.com/minio/minio-go/v7 go 1.23.0 toolchain go1.24.3 require ( github.com/dustin/go-humanize v1.0.1 github.com/go-ini/ini v1.67.0 github.com/google/uuid v1.6.0 github.com/klauspost/compress v1.18.0 github.com/klauspost/crc32 v1.3.0 github.com/minio/crc64nvme v1.1.0 github.com/minio/md5-simd v1.1.2 github.com/rs/xid v1.6.0 github.com/tinylib/msgp v1.3.0 golang.org/x/crypto v0.36.0 golang.org/x/net v0.38.0 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/klauspost/cpuid/v2 v2.2.11 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/stretchr/testify v1.9.0 // indirect golang.org/x/sys v0.34.0 // indirect golang.org/x/text v0.26.0 // indirect ) minio-go-7.0.97/go.sum000066400000000000000000000065721510244170000144730ustar00rootroot00000000000000github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q= github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= minio-go-7.0.97/healthcheck_test.go000066400000000000000000000033061510244170000171610ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "math/rand" "net/http" "net/http/httptest" "testing" "time" ) func TestHealthCheck(t *testing.T) { srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) defer srv.Close() // New - instantiate minio client with options clnt, err := New(srv.Listener.Addr().String(), &Options{ Region: "us-east-1", }) if err != nil { t.Fatal(err) } hcancel, err := clnt.HealthCheck(1 * time.Second) if err != nil { t.Fatal(err) } probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-health-") ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() clnt.BucketExists(ctx, probeBucketName) if !clnt.IsOffline() { t.Fatal("Expected offline but found online") } srv.Start() time.Sleep(2 * time.Second) if clnt.IsOffline() { t.Fatal("Expected online but found offline") } hcancel() // healthcheck is canceled. if !clnt.IsOnline() { t.Fatal("Expected online but found offline") } } minio-go-7.0.97/hook-reader.go000066400000000000000000000047741510244170000160710ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "fmt" "io" ) // hookReader hooks additional reader in the source stream. It is // useful for making progress bars. Second reader is appropriately // notified about the exact number of bytes read from the primary // source on each Read operation. type hookReader struct { source io.Reader hook io.Reader } // Seek implements io.Seeker. Seeks source first, and if necessary // seeks hook if Seek method is appropriately found. func (hr *hookReader) Seek(offset int64, whence int) (n int64, err error) { // Verify for source has embedded Seeker, use it. sourceSeeker, ok := hr.source.(io.Seeker) if ok { n, err = sourceSeeker.Seek(offset, whence) if err != nil { return 0, err } } if hr.hook != nil { // Verify if hook has embedded Seeker, use it. hookSeeker, ok := hr.hook.(io.Seeker) if ok { var m int64 m, err = hookSeeker.Seek(offset, whence) if err != nil { return 0, err } if n != m { return 0, fmt.Errorf("hook seeker seeked %d bytes, expected source %d bytes", m, n) } } } return n, nil } // Read implements io.Reader. Always reads from the source, the return // value 'n' number of bytes are reported through the hook. Returns // error for all non io.EOF conditions. func (hr *hookReader) Read(b []byte) (n int, err error) { n, err = hr.source.Read(b) if err != nil && err != io.EOF { return n, err } if hr.hook != nil { // Progress the hook with the total read bytes from the source. if _, herr := hr.hook.Read(b[:n]); herr != nil { if herr != io.EOF { return n, herr } } } return n, err } // newHook returns a io.ReadSeeker which implements hookReader that // reports the data read from the source to the hook. func newHook(source, hook io.Reader) io.Reader { if hook == nil { return &hookReader{source: source} } return &hookReader{ source: source, hook: hook, } } minio-go-7.0.97/pkg/000077500000000000000000000000001510244170000141075ustar00rootroot00000000000000minio-go-7.0.97/pkg/cors/000077500000000000000000000000001510244170000150555ustar00rootroot00000000000000minio-go-7.0.97/pkg/cors/cors.go000066400000000000000000000052061510244170000163550ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2024 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package cors import ( "encoding/xml" "fmt" "io" "strings" "github.com/dustin/go-humanize" ) const defaultXMLNS = "http://s3.amazonaws.com/doc/2006-03-01/" // Config is the container for a CORS configuration for a bucket. type Config struct { XMLNS string `xml:"xmlns,attr,omitempty"` XMLName xml.Name `xml:"CORSConfiguration"` CORSRules []Rule `xml:"CORSRule"` } // Rule is a single rule in a CORS configuration. type Rule struct { AllowedHeader []string `xml:"AllowedHeader,omitempty"` AllowedMethod []string `xml:"AllowedMethod,omitempty"` AllowedOrigin []string `xml:"AllowedOrigin,omitempty"` ExposeHeader []string `xml:"ExposeHeader,omitempty"` ID string `xml:"ID,omitempty"` MaxAgeSeconds int `xml:"MaxAgeSeconds,omitempty"` } // NewConfig creates a new CORS configuration with the given rules. func NewConfig(rules []Rule) *Config { return &Config{ XMLNS: defaultXMLNS, XMLName: xml.Name{ Local: "CORSConfiguration", Space: defaultXMLNS, }, CORSRules: rules, } } // ParseBucketCorsConfig parses a CORS configuration in XML from an io.Reader. func ParseBucketCorsConfig(reader io.Reader) (*Config, error) { var c Config // Max size of cors document is 64KiB according to https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketCors.html // This limiter is just for safety so has a max of 128KiB err := xml.NewDecoder(io.LimitReader(reader, 128*humanize.KiByte)).Decode(&c) if err != nil { return nil, fmt.Errorf("decoding xml: %w", err) } if c.XMLNS == "" { c.XMLNS = defaultXMLNS } for i, rule := range c.CORSRules { for j, method := range rule.AllowedMethod { c.CORSRules[i].AllowedMethod[j] = strings.ToUpper(method) } } return &c, nil } // ToXML marshals the CORS configuration to XML. func (c Config) ToXML() ([]byte, error) { if c.XMLNS == "" { c.XMLNS = defaultXMLNS } data, err := xml.Marshal(&c) if err != nil { return nil, fmt.Errorf("marshaling xml: %w", err) } return append([]byte(xml.Header), data...), nil } minio-go-7.0.97/pkg/cors/cors_test.go000066400000000000000000000022671510244170000174200ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2024 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package cors import ( "bytes" "os" "testing" ) func TestCORSXMLMarshal(t *testing.T) { fileContents, err := os.ReadFile("testdata/example.xml") if err != nil { t.Fatal(err) } c, err := ParseBucketCorsConfig(bytes.NewReader(fileContents)) if err != nil { t.Fatal(err) } remarshalled, err := c.ToXML() if err != nil { t.Fatal(err) } trimmedFileContents := bytes.TrimSpace(fileContents) if !bytes.Equal(trimmedFileContents, remarshalled) { t.Errorf("got: %s, want: %s", string(remarshalled), string(trimmedFileContents)) } } minio-go-7.0.97/pkg/cors/testdata/000077500000000000000000000000001510244170000166665ustar00rootroot00000000000000minio-go-7.0.97/pkg/cors/testdata/example.xml000066400000000000000000000014671510244170000210530ustar00rootroot00000000000000 *PUTPOSTDELETEhttp://www.example1.com*PUTPOSTDELETEhttp://www.example2.*GET*x-amz-id-26000POSThttps://www.example3.com minio-go-7.0.97/pkg/credentials/000077500000000000000000000000001510244170000164045ustar00rootroot00000000000000minio-go-7.0.97/pkg/credentials/assume_role.go000066400000000000000000000205441510244170000212560ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "bytes" "crypto/sha256" "encoding/hex" "encoding/xml" "errors" "io" "net/http" "net/url" "strconv" "strings" "time" "github.com/minio/minio-go/v7/pkg/signer" ) // AssumeRoleResponse contains the result of successful AssumeRole request. type AssumeRoleResponse struct { XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleResponse" json:"-"` Result AssumeRoleResult `xml:"AssumeRoleResult"` ResponseMetadata struct { RequestID string `xml:"RequestId,omitempty"` } `xml:"ResponseMetadata,omitempty"` } // AssumeRoleResult - Contains the response to a successful AssumeRole // request, including temporary credentials that can be used to make // MinIO API requests. type AssumeRoleResult struct { // The identifiers for the temporary security credentials that the operation // returns. AssumedRoleUser AssumedRoleUser `xml:",omitempty"` // The temporary security credentials, which include an access key ID, a secret // access key, and a security (or session) token. // // Note: The size of the security token that STS APIs return is not fixed. We // strongly recommend that you make no assumptions about the maximum size. As // of this writing, the typical size is less than 4096 bytes, but that can vary. // Also, future updates to AWS might require larger sizes. Credentials struct { AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"` SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"` Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"` SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"` } `xml:",omitempty"` // A percentage value that indicates the size of the policy in packed form. // The service rejects any policy with a packed size greater than 100 percent, // which means the policy exceeded the allowed space. PackedPolicySize int `xml:",omitempty"` } // A STSAssumeRole retrieves credentials from MinIO service, and keeps track if // those credentials are expired. type STSAssumeRole struct { Expiry // Optional http Client to use when connecting to MinIO STS service // (overrides default client in CredContext) Client *http.Client // STS endpoint to fetch STS credentials. STSEndpoint string // various options for this request. Options STSAssumeRoleOptions } // STSAssumeRoleOptions collection of various input options // to obtain AssumeRole credentials. type STSAssumeRoleOptions struct { // Mandatory inputs. AccessKey string SecretKey string SessionToken string // Optional if the first request is made with temporary credentials. Policy string // Optional to assign a policy to the assumed role Location string // Optional commonly needed with AWS STS. DurationSeconds int // Optional defaults to 1 hour. // Optional only valid if using with AWS STS RoleARN string RoleSessionName string ExternalID string TokenRevokeType string // Optional, used for token revokation (MinIO only extension) } // NewSTSAssumeRole returns a pointer to a new // Credentials object wrapping the STSAssumeRole. func NewSTSAssumeRole(stsEndpoint string, opts STSAssumeRoleOptions) (*Credentials, error) { if opts.AccessKey == "" || opts.SecretKey == "" { return nil, errors.New("AssumeRole credentials access/secretkey is mandatory") } return New(&STSAssumeRole{ STSEndpoint: stsEndpoint, Options: opts, }), nil } const defaultDurationSeconds = 3600 // closeResponse close non nil response with any response Body. // convenient wrapper to drain any remaining data on response body. // // Subsequently this allows golang http RoundTripper // to re-use the same connection for future requests. func closeResponse(resp *http.Response) { // Callers should close resp.Body when done reading from it. // If resp.Body is not closed, the Client's underlying RoundTripper // (typically Transport) may not be able to re-use a persistent TCP // connection to the server for a subsequent "keep-alive" request. if resp != nil && resp.Body != nil { // Drain any remaining Body and then close the connection. // Without this closing connection would disallow re-using // the same connection for future uses. // - http://stackoverflow.com/a/17961593/4465767 io.Copy(io.Discard, resp.Body) resp.Body.Close() } } func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssumeRoleOptions) (AssumeRoleResponse, error) { v := url.Values{} v.Set("Action", "AssumeRole") v.Set("Version", STSVersion) if opts.RoleARN != "" { v.Set("RoleArn", opts.RoleARN) } if opts.RoleSessionName != "" { v.Set("RoleSessionName", opts.RoleSessionName) } if opts.DurationSeconds > defaultDurationSeconds { v.Set("DurationSeconds", strconv.Itoa(opts.DurationSeconds)) } else { v.Set("DurationSeconds", strconv.Itoa(defaultDurationSeconds)) } if opts.Policy != "" { v.Set("Policy", opts.Policy) } if opts.ExternalID != "" { v.Set("ExternalId", opts.ExternalID) } if opts.TokenRevokeType != "" { v.Set("TokenRevokeType", opts.TokenRevokeType) } u, err := url.Parse(endpoint) if err != nil { return AssumeRoleResponse{}, err } u.Path = "/" postBody := strings.NewReader(v.Encode()) hash := sha256.New() if _, err = io.Copy(hash, postBody); err != nil { return AssumeRoleResponse{}, err } postBody.Seek(0, 0) req, err := http.NewRequest(http.MethodPost, u.String(), postBody) if err != nil { return AssumeRoleResponse{}, err } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(hash.Sum(nil))) if opts.SessionToken != "" { req.Header.Set("X-Amz-Security-Token", opts.SessionToken) } req = signer.SignV4STS(*req, opts.AccessKey, opts.SecretKey, opts.Location) resp, err := clnt.Do(req) if err != nil { return AssumeRoleResponse{}, err } defer closeResponse(resp) if resp.StatusCode != http.StatusOK { var errResp ErrorResponse buf, err := io.ReadAll(resp.Body) if err != nil { return AssumeRoleResponse{}, err } _, err = xmlDecodeAndBody(bytes.NewReader(buf), &errResp) if err != nil { var s3Err Error if _, err = xmlDecodeAndBody(bytes.NewReader(buf), &s3Err); err != nil { return AssumeRoleResponse{}, err } errResp.RequestID = s3Err.RequestID errResp.STSError.Code = s3Err.Code errResp.STSError.Message = s3Err.Message } return AssumeRoleResponse{}, errResp } a := AssumeRoleResponse{} if _, err = xmlDecodeAndBody(resp.Body, &a); err != nil { return AssumeRoleResponse{}, err } return a, nil } // RetrieveWithCredContext retrieves credentials from the MinIO service. // Error will be returned if the request fails, optional cred context. func (m *STSAssumeRole) RetrieveWithCredContext(cc *CredContext) (Value, error) { if cc == nil { cc = defaultCredContext } client := m.Client if client == nil { client = cc.Client } if client == nil { client = defaultCredContext.Client } stsEndpoint := m.STSEndpoint if stsEndpoint == "" { stsEndpoint = cc.Endpoint } if stsEndpoint == "" { return Value{}, errors.New("STS endpoint unknown") } a, err := getAssumeRoleCredentials(client, stsEndpoint, m.Options) if err != nil { return Value{}, err } // Expiry window is set to 10secs. m.SetExpiration(a.Result.Credentials.Expiration, DefaultExpiryWindow) return Value{ AccessKeyID: a.Result.Credentials.AccessKey, SecretAccessKey: a.Result.Credentials.SecretKey, SessionToken: a.Result.Credentials.SessionToken, Expiration: a.Result.Credentials.Expiration, SignerType: SignatureV4, }, nil } // Retrieve retrieves credentials from the MinIO service. // Error will be returned if the request fails. func (m *STSAssumeRole) Retrieve() (Value, error) { return m.RetrieveWithCredContext(nil) } minio-go-7.0.97/pkg/credentials/chain.go000066400000000000000000000066051510244170000200240ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials // A Chain will search for a provider which returns credentials // and cache that provider until Retrieve is called again. // // The Chain provides a way of chaining multiple providers together // which will pick the first available using priority order of the // Providers in the list. // // If none of the Providers retrieve valid credentials Value, ChainProvider's // Retrieve() will return the no credentials value. // // If a Provider is found which returns valid credentials Value ChainProvider // will cache that Provider for all calls to IsExpired(), until Retrieve is // called again after IsExpired() is true. // // creds := credentials.NewChainCredentials( // []credentials.Provider{ // &credentials.EnvAWSS3{}, // &credentials.EnvMinio{}, // }) // // // Usage of ChainCredentials. // mc, err := minio.NewWithCredentials(endpoint, creds, secure, "us-east-1") // if err != nil { // log.Fatalln(err) // } type Chain struct { Providers []Provider curr Provider } // NewChainCredentials returns a pointer to a new Credentials object // wrapping a chain of providers. func NewChainCredentials(providers []Provider) *Credentials { return New(&Chain{ Providers: append([]Provider{}, providers...), }) } // RetrieveWithCredContext is like Retrieve with CredContext func (c *Chain) RetrieveWithCredContext(cc *CredContext) (Value, error) { for _, p := range c.Providers { creds, _ := p.RetrieveWithCredContext(cc) // Always prioritize non-anonymous providers, if any. if creds.AccessKeyID == "" && creds.SecretAccessKey == "" { continue } c.curr = p return creds, nil } // At this point we have exhausted all the providers and // are left without any credentials return anonymous. return Value{ SignerType: SignatureAnonymous, }, nil } // Retrieve returns the credentials value, returns no credentials(anonymous) // if no credentials provider returned any value. // // If a provider is found with credentials, it will be cached and any calls // to IsExpired() will return the expired state of the cached provider. func (c *Chain) Retrieve() (Value, error) { for _, p := range c.Providers { creds, _ := p.Retrieve() // Always prioritize non-anonymous providers, if any. if creds.AccessKeyID == "" && creds.SecretAccessKey == "" { continue } c.curr = p return creds, nil } // At this point we have exhausted all the providers and // are left without any credentials return anonymous. return Value{ SignerType: SignatureAnonymous, }, nil } // IsExpired will returned the expired state of the currently cached provider // if there is one. If there is no current provider, true will be returned. func (c *Chain) IsExpired() bool { if c.curr != nil { return c.curr.IsExpired() } return true } minio-go-7.0.97/pkg/credentials/chain_test.go000066400000000000000000000063611510244170000210620ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "errors" "testing" ) type testCredProvider struct { creds Value expired bool err error } func (s *testCredProvider) Retrieve() (Value, error) { s.expired = false return s.creds, s.err } func (s *testCredProvider) RetrieveWithCredContext(_ *CredContext) (Value, error) { s.expired = false return s.creds, s.err } func (s *testCredProvider) IsExpired() bool { return s.expired } func TestChainGet(t *testing.T) { p := &Chain{ Providers: []Provider{ &credProvider{err: errors.New("FirstError")}, &credProvider{err: errors.New("SecondError")}, &testCredProvider{ creds: Value{ AccessKeyID: "AKIF", SecretAccessKey: "NOSECRET", SessionToken: "", }, }, &credProvider{ creds: Value{ AccessKeyID: "AKID", SecretAccessKey: "SECRET", SessionToken: "", }, }, }, } creds, err := p.RetrieveWithCredContext(defaultCredContext) if err != nil { t.Fatal(err) } // Also check credentials if creds.AccessKeyID != "AKIF" { t.Fatalf("Expected 'AKIF', got %s", creds.AccessKeyID) } if creds.SecretAccessKey != "NOSECRET" { t.Fatalf("Expected 'NOSECRET', got %s", creds.SecretAccessKey) } if creds.SessionToken != "" { t.Fatalf("Expected empty token, got %s", creds.SessionToken) } } func TestChainIsExpired(t *testing.T) { credProvider := &credProvider{ creds: Value{ AccessKeyID: "UXHW", SecretAccessKey: "MYSECRET", SessionToken: "", }, expired: true, } p := &Chain{ Providers: []Provider{ credProvider, }, } if !p.IsExpired() { t.Fatal("Expected expired to be true before any Retrieve") } _, err := p.RetrieveWithCredContext(defaultCredContext) if err != nil { t.Fatal(err) } if p.IsExpired() { t.Fatal("Expected to be not expired after Retrieve") } } func TestChainWithNoProvider(t *testing.T) { p := &Chain{ Providers: []Provider{}, } if !p.IsExpired() { t.Fatal("Expected to be expired with no providers") } _, err := p.RetrieveWithCredContext(defaultCredContext) if err != nil { if err.Error() != "No valid providers found []" { t.Error(err) } } } func TestChainProviderWithNoValidProvider(t *testing.T) { errs := []error{ errors.New("FirstError"), errors.New("SecondError"), } p := &Chain{ Providers: []Provider{ &credProvider{err: errs[0]}, &credProvider{err: errs[1]}, }, } if !p.IsExpired() { t.Fatal("Expected to be expired with no providers") } _, err := p.RetrieveWithCredContext(defaultCredContext) if err != nil { if err.Error() != "No valid providers found [FirstError SecondError]" { t.Error(err) } } } minio-go-7.0.97/pkg/credentials/config.json.sample000066400000000000000000000005061510244170000220250ustar00rootroot00000000000000{ "version": "8", "hosts": { "play": { "url": "https://play.min.io", "accessKey": "Q3AM3UQ867SPQQA43P2F", "secretKey": "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "api": "S3v2" }, "s3": { "url": "https://s3.amazonaws.com", "accessKey": "accessKey", "secretKey": "secret", "api": "S3v4" } } }minio-go-7.0.97/pkg/credentials/credentials.go000066400000000000000000000164331510244170000212370ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "net/http" "sync" "time" ) const ( // STSVersion sts version string STSVersion = "2011-06-15" // How much duration to slash from the given expiration duration defaultExpiryWindow = 0.8 ) // defaultCredContext is used when the credential context doesn't // actually matter or the default context is suitable. var defaultCredContext = &CredContext{Client: http.DefaultClient} // A Value is the S3 credentials value for individual credential fields. type Value struct { // S3 Access key ID AccessKeyID string // S3 Secret Access Key SecretAccessKey string // S3 Session Token SessionToken string // Expiration of this credentials - null means no expiration associated Expiration time.Time // Signature Type. SignerType SignatureType } // A Provider is the interface for any component which will provide credentials // Value. A provider is required to manage its own Expired state, and what to // be expired means. type Provider interface { // RetrieveWithCredContext returns nil if it successfully retrieved the // value. Error is returned if the value were not obtainable, or empty. // optionally takes CredContext for additional context to retrieve credentials. RetrieveWithCredContext(cc *CredContext) (Value, error) // Retrieve returns nil if it successfully retrieved the value. // Error is returned if the value were not obtainable, or empty. // // Deprecated: Retrieve() exists for historical compatibility and should not // be used. To get new credentials use the RetrieveWithCredContext function // to ensure the proper context (i.e. HTTP client) will be used. Retrieve() (Value, error) // IsExpired returns if the credentials are no longer valid, and need // to be retrieved. IsExpired() bool } // CredContext is passed to the Retrieve function of a provider to provide // some additional context to retrieve credentials. type CredContext struct { // Client specifies the HTTP client that should be used if an HTTP // request is to be made to fetch the credentials. Client *http.Client // Endpoint specifies the MinIO endpoint that will be used if no // explicit endpoint is provided. Endpoint string } // A Expiry provides shared expiration logic to be used by credentials // providers to implement expiry functionality. // // The best method to use this struct is as an anonymous field within the // provider's struct. // // Example: // // type IAMCredentialProvider struct { // Expiry // ... // } type Expiry struct { // The date/time when to expire on expiration time.Time // If set will be used by IsExpired to determine the current time. // Defaults to time.Now if CurrentTime is not set. CurrentTime func() time.Time } // SetExpiration sets the expiration IsExpired will check when called. // // If window is greater than 0 the expiration time will be reduced by the // window value. // // Using a window is helpful to trigger credentials to expire sooner than // the expiration time given to ensure no requests are made with expired // tokens. func (e *Expiry) SetExpiration(expiration time.Time, window time.Duration) { if e.CurrentTime == nil { e.CurrentTime = time.Now } cut := window if cut < 0 { expireIn := expiration.Sub(e.CurrentTime()) cut = time.Duration(float64(expireIn) * (1 - defaultExpiryWindow)) } e.expiration = expiration.Add(-cut) } // IsExpired returns if the credentials are expired. func (e *Expiry) IsExpired() bool { if e.CurrentTime == nil { e.CurrentTime = time.Now } return e.expiration.Before(e.CurrentTime()) } // Credentials - A container for synchronous safe retrieval of credentials Value. // Credentials will cache the credentials value until they expire. Once the value // expires the next Get will attempt to retrieve valid credentials. // // Credentials is safe to use across multiple goroutines and will manage the // synchronous state so the Providers do not need to implement their own // synchronization. // // The first Credentials.Get() will always call Provider.Retrieve() to get the // first instance of the credentials Value. All calls to Get() after that // will return the cached credentials Value until IsExpired() returns true. type Credentials struct { sync.Mutex creds Value forceRefresh bool provider Provider } // New returns a pointer to a new Credentials with the provider set. func New(provider Provider) *Credentials { return &Credentials{ provider: provider, forceRefresh: true, } } // Get returns the credentials value, or error if the credentials Value failed // to be retrieved. // // Will return the cached credentials Value if it has not expired. If the // credentials Value has expired the Provider's Retrieve() will be called // to refresh the credentials. // // If Credentials.Expire() was called the credentials Value will be force // expired, and the next call to Get() will cause them to be refreshed. // // Deprecated: Get() exists for historical compatibility and should not be // used. To get new credentials use the Credentials.GetWithContext function // to ensure the proper context (i.e. HTTP client) will be used. func (c *Credentials) Get() (Value, error) { return c.GetWithContext(nil) } // GetWithContext returns the credentials value, or error if the // credentials Value failed to be retrieved. // // Will return the cached credentials Value if it has not expired. If the // credentials Value has expired the Provider's Retrieve() will be called // to refresh the credentials. // // If Credentials.Expire() was called the credentials Value will be force // expired, and the next call to Get() will cause them to be refreshed. func (c *Credentials) GetWithContext(cc *CredContext) (Value, error) { if c == nil { return Value{}, nil } if cc == nil { cc = defaultCredContext } c.Lock() defer c.Unlock() if c.isExpired() { creds, err := c.provider.RetrieveWithCredContext(cc) if err != nil { return Value{}, err } c.creds = creds c.forceRefresh = false } return c.creds, nil } // Expire expires the credentials and forces them to be retrieved on the // next call to Get(). // // This will override the Provider's expired state, and force Credentials // to call the Provider's Retrieve(). func (c *Credentials) Expire() { c.Lock() defer c.Unlock() c.forceRefresh = true } // IsExpired returns if the credentials are no longer valid, and need // to be refreshed. // // If the Credentials were forced to be expired with Expire() this will // reflect that override. func (c *Credentials) IsExpired() bool { c.Lock() defer c.Unlock() return c.isExpired() } // isExpired helper method wrapping the definition of expired credentials. func (c *Credentials) isExpired() bool { return c.forceRefresh || c.provider.IsExpired() } minio-go-7.0.97/pkg/credentials/credentials.json000066400000000000000000000002411510244170000215710ustar00rootroot00000000000000{ "Version": 1, "SessionToken": "token", "AccessKeyId": "accessKey", "SecretAccessKey": "secret", "Expiration": "9999-04-27T16:02:25.000Z" } minio-go-7.0.97/pkg/credentials/credentials.sample000066400000000000000000000004621510244170000221060ustar00rootroot00000000000000[default] aws_access_key_id = accessKey aws_secret_access_key = secret aws_session_token = token [no_token] aws_access_key_id = accessKey aws_secret_access_key = secret [with_colon] aws_access_key_id: accessKey aws_secret_access_key: secret [with_process] credential_process = /bin/cat credentials.json minio-go-7.0.97/pkg/credentials/credentials_test.go000066400000000000000000000036351510244170000222760ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "errors" "testing" ) type credProvider struct { creds Value expired bool err error } func (s *credProvider) Retrieve() (Value, error) { s.expired = false return s.creds, s.err } func (s *credProvider) RetrieveWithCredContext(_ *CredContext) (Value, error) { s.expired = false return s.creds, s.err } func (s *credProvider) IsExpired() bool { return s.expired } func TestCredentialsGet(t *testing.T) { c := New(&credProvider{ creds: Value{ AccessKeyID: "UXHW", SecretAccessKey: "MYSECRET", SessionToken: "", }, expired: true, }) creds, err := c.GetWithContext(defaultCredContext) if err != nil { t.Fatal(err) } if creds.AccessKeyID != "UXHW" { t.Errorf("Expected \"UXHW\", got %s", creds.AccessKeyID) } if creds.SecretAccessKey != "MYSECRET" { t.Errorf("Expected \"MYSECRET\", got %s", creds.SecretAccessKey) } if creds.SessionToken != "" { t.Errorf("Expected session token to be empty, got %s", creds.SessionToken) } } func TestCredentialsGetWithError(t *testing.T) { c := New(&credProvider{err: errors.New("Custom error")}) _, err := c.GetWithContext(defaultCredContext) if err != nil { if err.Error() != "Custom error" { t.Errorf("Expected \"Custom error\", got %s", err.Error()) } } } minio-go-7.0.97/pkg/credentials/doc.go000066400000000000000000000043551510244170000175070ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // Package credentials provides credential retrieval and management // for S3 compatible object storage. // // By default the Credentials.Get() will cache the successful result of a // Provider's Retrieve() until Provider.IsExpired() returns true. At which // point Credentials will call Provider's Retrieve() to get new credential Value. // // The Provider is responsible for determining when credentials have expired. // It is also important to note that Credentials will always call Retrieve the // first time Credentials.Get() is called. // // Example of using the environment variable credentials. // // creds := NewFromEnv() // // Retrieve the credentials value // credValue, err := creds.Get() // if err != nil { // // handle error // } // // Example of forcing credentials to expire and be refreshed on the next Get(). // This may be helpful to proactively expire credentials and refresh them sooner // than they would naturally expire on their own. // // creds := NewFromIAM("") // creds.Expire() // credsValue, err := creds.Get() // // New credentials will be retrieved instead of from cache. // // # Custom Provider // // Each Provider built into this package also provides a helper method to generate // a Credentials pointer setup with the provider. To use a custom Provider just // create a type which satisfies the Provider interface and pass it to the // NewCredentials method. // // type MyProvider struct{} // func (m *MyProvider) Retrieve() (Value, error) {...} // func (m *MyProvider) IsExpired() bool {...} // // creds := NewCredentials(&MyProvider{}) // credValue, err := creds.Get() package credentials minio-go-7.0.97/pkg/credentials/env_aws.go000066400000000000000000000042161510244170000204000ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import "os" // A EnvAWS retrieves credentials from the environment variables of the // running process. EnvAWSironment credentials never expire. // // EnvAWSironment variables used: // // * Access Key ID: AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY. // * Secret Access Key: AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY. // * Secret Token: AWS_SESSION_TOKEN. type EnvAWS struct { retrieved bool } // NewEnvAWS returns a pointer to a new Credentials object // wrapping the environment variable provider. func NewEnvAWS() *Credentials { return New(&EnvAWS{}) } func (e *EnvAWS) retrieve() (Value, error) { e.retrieved = false id := os.Getenv("AWS_ACCESS_KEY_ID") if id == "" { id = os.Getenv("AWS_ACCESS_KEY") } secret := os.Getenv("AWS_SECRET_ACCESS_KEY") if secret == "" { secret = os.Getenv("AWS_SECRET_KEY") } signerType := SignatureV4 if id == "" || secret == "" { signerType = SignatureAnonymous } e.retrieved = true return Value{ AccessKeyID: id, SecretAccessKey: secret, SessionToken: os.Getenv("AWS_SESSION_TOKEN"), SignerType: signerType, }, nil } // Retrieve retrieves the keys from the environment. func (e *EnvAWS) Retrieve() (Value, error) { return e.retrieve() } // RetrieveWithCredContext is like Retrieve (no-op input of Cred Context) func (e *EnvAWS) RetrieveWithCredContext(_ *CredContext) (Value, error) { return e.retrieve() } // IsExpired returns if the credentials have been retrieved. func (e *EnvAWS) IsExpired() bool { return !e.retrieved } minio-go-7.0.97/pkg/credentials/env_minio.go000066400000000000000000000041501510244170000207160ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import "os" // A EnvMinio retrieves credentials from the environment variables of the // running process. EnvMinioironment credentials never expire. // // Environment variables used: // // * Access Key ID: MINIO_ACCESS_KEY. // * Secret Access Key: MINIO_SECRET_KEY. // * Access Key ID: MINIO_ROOT_USER. // * Secret Access Key: MINIO_ROOT_PASSWORD. type EnvMinio struct { retrieved bool } // NewEnvMinio returns a pointer to a new Credentials object // wrapping the environment variable provider. func NewEnvMinio() *Credentials { return New(&EnvMinio{}) } func (e *EnvMinio) retrieve() (Value, error) { e.retrieved = false id := os.Getenv("MINIO_ROOT_USER") secret := os.Getenv("MINIO_ROOT_PASSWORD") signerType := SignatureV4 if id == "" || secret == "" { id = os.Getenv("MINIO_ACCESS_KEY") secret = os.Getenv("MINIO_SECRET_KEY") if id == "" || secret == "" { signerType = SignatureAnonymous } } e.retrieved = true return Value{ AccessKeyID: id, SecretAccessKey: secret, SignerType: signerType, }, nil } // Retrieve retrieves the keys from the environment. func (e *EnvMinio) Retrieve() (Value, error) { return e.retrieve() } // RetrieveWithCredContext is like Retrieve() (no-op input cred context) func (e *EnvMinio) RetrieveWithCredContext(_ *CredContext) (Value, error) { return e.retrieve() } // IsExpired returns if the credentials have been retrieved. func (e *EnvMinio) IsExpired() bool { return !e.retrieved } minio-go-7.0.97/pkg/credentials/env_test.go000066400000000000000000000046711510244170000205720ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "os" "reflect" "testing" ) func TestEnvAWSRetrieve(t *testing.T) { os.Clearenv() t.Setenv("AWS_ACCESS_KEY_ID", "access") t.Setenv("AWS_SECRET_ACCESS_KEY", "secret") t.Setenv("AWS_SESSION_TOKEN", "token") e := EnvAWS{} if !e.IsExpired() { t.Error("Expect creds to be expired before retrieve.") } creds, err := e.RetrieveWithCredContext(defaultCredContext) if err != nil { t.Fatal(err) } expectedCreds := Value{ AccessKeyID: "access", SecretAccessKey: "secret", SessionToken: "token", SignerType: SignatureV4, } if !reflect.DeepEqual(creds, expectedCreds) { t.Errorf("Expected %v, got %v", expectedCreds, creds) } if e.IsExpired() { t.Error("Expect creds to not be expired after retrieve.") } os.Clearenv() t.Setenv("AWS_ACCESS_KEY", "access") t.Setenv("AWS_SECRET_KEY", "secret") expectedCreds = Value{ AccessKeyID: "access", SecretAccessKey: "secret", SignerType: SignatureV4, } creds, err = e.RetrieveWithCredContext(defaultCredContext) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(creds, expectedCreds) { t.Errorf("Expected %v, got %v", expectedCreds, creds) } } func TestEnvMinioRetrieve(t *testing.T) { os.Clearenv() t.Setenv("MINIO_ACCESS_KEY", "access") t.Setenv("MINIO_SECRET_KEY", "secret") e := EnvMinio{} if !e.IsExpired() { t.Error("Expect creds to be expired before retrieve.") } creds, err := e.RetrieveWithCredContext(defaultCredContext) if err != nil { t.Fatal(err) } expectedCreds := Value{ AccessKeyID: "access", SecretAccessKey: "secret", SignerType: SignatureV4, } if !reflect.DeepEqual(creds, expectedCreds) { t.Errorf("Expected %v, got %v", expectedCreds, creds) } if e.IsExpired() { t.Error("Expect creds to not be expired after retrieve.") } } minio-go-7.0.97/pkg/credentials/error_response.go000066400000000000000000000053621510244170000220100ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "bytes" "encoding/xml" "fmt" "io" ) // ErrorResponse - Is the typed error returned. // ErrorResponse struct should be comparable since it is compared inside // golang http API (https://github.com/golang/go/issues/29768) type ErrorResponse struct { XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ ErrorResponse" json:"-"` STSError struct { Type string `xml:"Type"` Code string `xml:"Code"` Message string `xml:"Message"` } `xml:"Error"` RequestID string `xml:"RequestId"` } // Error - Is the typed error returned by all API operations. type Error struct { XMLName xml.Name `xml:"Error" json:"-"` Code string Message string BucketName string Key string Resource string RequestID string `xml:"RequestId"` HostID string `xml:"HostId"` // Region where the bucket is located. This header is returned // only in HEAD bucket and ListObjects response. Region string // Captures the server string returned in response header. Server string // Underlying HTTP status code for the returned error StatusCode int `xml:"-" json:"-"` } // Error - Returns S3 error string. func (e Error) Error() string { if e.Message == "" { return fmt.Sprintf("Error response code %s.", e.Code) } return e.Message } // Error - Returns STS error string. func (e ErrorResponse) Error() string { if e.STSError.Message == "" { return fmt.Sprintf("Error response code %s.", e.STSError.Code) } return e.STSError.Message } // xmlDecoder provide decoded value in xml. func xmlDecoder(body io.Reader, v interface{}) error { d := xml.NewDecoder(body) return d.Decode(v) } // xmlDecodeAndBody reads the whole body up to 1MB and // tries to XML decode it into v. // The body that was read and any error from reading or decoding is returned. func xmlDecodeAndBody(bodyReader io.Reader, v interface{}) ([]byte, error) { // read the whole body (up to 1MB) const maxBodyLength = 1 << 20 body, err := io.ReadAll(io.LimitReader(bodyReader, maxBodyLength)) if err != nil { return nil, err } return bytes.TrimSpace(body), xmlDecoder(bytes.NewReader(body), v) } minio-go-7.0.97/pkg/credentials/file_aws_credentials.go000066400000000000000000000117121510244170000231030ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "encoding/json" "errors" "os" "os/exec" "path/filepath" "strings" "time" "github.com/go-ini/ini" ) // A externalProcessCredentials stores the output of a credential_process type externalProcessCredentials struct { Version int SessionToken string AccessKeyID string `json:"AccessKeyId"` SecretAccessKey string Expiration time.Time } // A FileAWSCredentials retrieves credentials from the current user's home // directory, and keeps track if those credentials are expired. // // Profile ini file example: $HOME/.aws/credentials type FileAWSCredentials struct { Expiry // Path to the shared credentials file. // // If empty will look for "AWS_SHARED_CREDENTIALS_FILE" env variable. If the // env value is empty will default to current user's home directory. // Linux/OSX: "$HOME/.aws/credentials" // Windows: "%USERPROFILE%\.aws\credentials" Filename string // AWS Profile to extract credentials from the shared credentials file. If empty // will default to environment variable "AWS_PROFILE" or "default" if // environment variable is also not set. Profile string // retrieved states if the credentials have been successfully retrieved. retrieved bool } // NewFileAWSCredentials returns a pointer to a new Credentials object // wrapping the Profile file provider. func NewFileAWSCredentials(filename, profile string) *Credentials { return New(&FileAWSCredentials{ Filename: filename, Profile: profile, }) } func (p *FileAWSCredentials) retrieve() (Value, error) { if p.Filename == "" { p.Filename = os.Getenv("AWS_SHARED_CREDENTIALS_FILE") if p.Filename == "" { homeDir, err := os.UserHomeDir() if err != nil { return Value{}, err } p.Filename = filepath.Join(homeDir, ".aws", "credentials") } } if p.Profile == "" { p.Profile = os.Getenv("AWS_PROFILE") if p.Profile == "" { p.Profile = "default" } } p.retrieved = false iniProfile, err := loadProfile(p.Filename, p.Profile) if err != nil { return Value{}, err } // Default to empty string if not found. id := iniProfile.Key("aws_access_key_id") // Default to empty string if not found. secret := iniProfile.Key("aws_secret_access_key") // Default to empty string if not found. token := iniProfile.Key("aws_session_token") // If credential_process is defined, obtain credentials by executing // the external process credentialProcess := strings.TrimSpace(iniProfile.Key("credential_process").String()) if credentialProcess != "" { args := strings.Fields(credentialProcess) if len(args) <= 1 { return Value{}, errors.New("invalid credential process args") } cmd := exec.Command(args[0], args[1:]...) out, err := cmd.Output() if err != nil { return Value{}, err } var externalProcessCredentials externalProcessCredentials err = json.Unmarshal([]byte(out), &externalProcessCredentials) if err != nil { return Value{}, err } p.retrieved = true p.SetExpiration(externalProcessCredentials.Expiration, DefaultExpiryWindow) return Value{ AccessKeyID: externalProcessCredentials.AccessKeyID, SecretAccessKey: externalProcessCredentials.SecretAccessKey, SessionToken: externalProcessCredentials.SessionToken, Expiration: externalProcessCredentials.Expiration, SignerType: SignatureV4, }, nil } p.retrieved = true return Value{ AccessKeyID: id.String(), SecretAccessKey: secret.String(), SessionToken: token.String(), SignerType: SignatureV4, }, nil } // Retrieve reads and extracts the shared credentials from the current // users home directory. func (p *FileAWSCredentials) Retrieve() (Value, error) { return p.retrieve() } // RetrieveWithCredContext is like Retrieve(), cred context is no-op for File credentials func (p *FileAWSCredentials) RetrieveWithCredContext(_ *CredContext) (Value, error) { return p.retrieve() } // loadProfiles loads from the file pointed to by shared credentials filename for profile. // The credentials retrieved from the profile will be returned or error. Error will be // returned if it fails to read from the file, or the data is invalid. func loadProfile(filename, profile string) (*ini.Section, error) { config, err := ini.Load(filename) if err != nil { return nil, err } iniProfile, err := config.GetSection(profile) if err != nil { return nil, err } return iniProfile, nil } minio-go-7.0.97/pkg/credentials/file_minio_client.go000066400000000000000000000077221510244170000224130ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "encoding/json" "os" "path/filepath" "runtime" ) // A FileMinioClient retrieves credentials from the current user's home // directory, and keeps track if those credentials are expired. // // Configuration file example: $HOME/.mc/config.json type FileMinioClient struct { // Path to the shared credentials file. // // If empty will look for "MINIO_SHARED_CREDENTIALS_FILE" env variable. If the // env value is empty will default to current user's home directory. // Linux/OSX: "$HOME/.mc/config.json" // Windows: "%USERALIAS%\mc\config.json" Filename string // MinIO Alias to extract credentials from the shared credentials file. If empty // will default to environment variable "MINIO_ALIAS" or "s3" if // environment variable is also not set. Alias string // retrieved states if the credentials have been successfully retrieved. retrieved bool } // NewFileMinioClient returns a pointer to a new Credentials object // wrapping the Alias file provider. func NewFileMinioClient(filename, alias string) *Credentials { return New(&FileMinioClient{ Filename: filename, Alias: alias, }) } func (p *FileMinioClient) retrieve() (Value, error) { if p.Filename == "" { if value, ok := os.LookupEnv("MINIO_SHARED_CREDENTIALS_FILE"); ok { p.Filename = value } else { homeDir, err := os.UserHomeDir() if err != nil { return Value{}, err } p.Filename = filepath.Join(homeDir, ".mc", "config.json") if runtime.GOOS == "windows" { p.Filename = filepath.Join(homeDir, "mc", "config.json") } } } if p.Alias == "" { p.Alias = os.Getenv("MINIO_ALIAS") if p.Alias == "" { p.Alias = "s3" } } p.retrieved = false hostCfg, err := loadAlias(p.Filename, p.Alias) if err != nil { return Value{}, err } p.retrieved = true return Value{ AccessKeyID: hostCfg.AccessKey, SecretAccessKey: hostCfg.SecretKey, SignerType: parseSignatureType(hostCfg.API), }, nil } // Retrieve reads and extracts the shared credentials from the current // users home directory. func (p *FileMinioClient) Retrieve() (Value, error) { return p.retrieve() } // RetrieveWithCredContext - is like Retrieve() func (p *FileMinioClient) RetrieveWithCredContext(_ *CredContext) (Value, error) { return p.retrieve() } // IsExpired returns if the shared credentials have expired. func (p *FileMinioClient) IsExpired() bool { return !p.retrieved } // hostConfig configuration of a host. type hostConfig struct { URL string `json:"url"` AccessKey string `json:"accessKey"` SecretKey string `json:"secretKey"` API string `json:"api"` } // config config version. type config struct { Version string `json:"version"` Hosts map[string]hostConfig `json:"hosts"` Aliases map[string]hostConfig `json:"aliases"` } // loadAliass loads from the file pointed to by shared credentials filename for alias. // The credentials retrieved from the alias will be returned or error. Error will be // returned if it fails to read from the file. func loadAlias(filename, alias string) (hostConfig, error) { cfg := &config{} configBytes, err := os.ReadFile(filename) if err != nil { return hostConfig{}, err } if err = json.Unmarshal(configBytes, cfg); err != nil { return hostConfig{}, err } if cfg.Version == "10" { return cfg.Aliases[alias], nil } return cfg.Hosts[alias], nil } minio-go-7.0.97/pkg/credentials/file_test.go000066400000000000000000000143171510244170000207170ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "os" "path/filepath" "runtime" "testing" ) func TestFileAWS(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("\"/bin/cat\": file does not exist") } os.Clearenv() creds := NewFileAWSCredentials("credentials.sample", "") credValues, err := creds.GetWithContext(defaultCredContext) if err != nil { t.Fatal(err) } if credValues.AccessKeyID != "accessKey" { t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) } if credValues.SecretAccessKey != "secret" { t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) } if credValues.SessionToken != "token" { t.Errorf("Expected 'token', got %s'", credValues.SessionToken) } t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "credentials.sample") creds = NewFileAWSCredentials("", "") credValues, err = creds.GetWithContext(defaultCredContext) if err != nil { t.Fatal(err) } if credValues.AccessKeyID != "accessKey" { t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) } if credValues.SecretAccessKey != "secret" { t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) } if credValues.SessionToken != "token" { t.Errorf("Expected 'token', got %s'", credValues.SessionToken) } wd, err := os.Getwd() if err != nil { t.Fatal(err) } t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(wd, "credentials.sample")) creds = NewFileAWSCredentials("", "") credValues, err = creds.GetWithContext(defaultCredContext) if err != nil { t.Fatal(err) } if credValues.AccessKeyID != "accessKey" { t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) } if credValues.SecretAccessKey != "secret" { t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) } if credValues.SessionToken != "token" { t.Errorf("Expected 'token', got %s'", credValues.SessionToken) } os.Clearenv() t.Setenv("AWS_PROFILE", "no_token") creds = NewFileAWSCredentials("credentials.sample", "") credValues, err = creds.GetWithContext(defaultCredContext) if err != nil { t.Fatal(err) } if credValues.AccessKeyID != "accessKey" { t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) } if credValues.SecretAccessKey != "secret" { t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) } os.Clearenv() creds = NewFileAWSCredentials("credentials.sample", "no_token") credValues, err = creds.GetWithContext(defaultCredContext) if err != nil { t.Fatal(err) } if credValues.AccessKeyID != "accessKey" { t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) } if credValues.SecretAccessKey != "secret" { t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) } creds = NewFileAWSCredentials("credentials-non-existent.sample", "no_token") _, err = creds.GetWithContext(defaultCredContext) if !os.IsNotExist(err) { t.Errorf("Expected open non-existent.json: no such file or directory, got %s", err) } if !creds.IsExpired() { t.Error("Should be expired if not loaded") } os.Clearenv() creds = NewFileAWSCredentials("credentials.sample", "with_process") credValues, err = creds.GetWithContext(defaultCredContext) if err != nil { t.Fatal(err) } if credValues.AccessKeyID != "accessKey" { t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) } if credValues.SecretAccessKey != "secret" { t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) } if credValues.SessionToken != "token" { t.Errorf("Expected 'token', got %s'", credValues.SessionToken) } if creds.IsExpired() { t.Error("Should not be expired") } } func TestFileMinioClient(t *testing.T) { os.Clearenv() creds := NewFileMinioClient("config.json.sample", "") credValues, err := creds.GetWithContext(defaultCredContext) if err != nil { t.Fatal(err) } if credValues.AccessKeyID != "accessKey" { t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) } if credValues.SecretAccessKey != "secret" { t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) } if credValues.SignerType != SignatureV4 { t.Errorf("Expected 'S3v4', got %s'", credValues.SignerType) } os.Clearenv() t.Setenv("MINIO_ALIAS", "play") creds = NewFileMinioClient("config.json.sample", "") credValues, err = creds.GetWithContext(defaultCredContext) if err != nil { t.Fatal(err) } if credValues.AccessKeyID != "Q3AM3UQ867SPQQA43P2F" { t.Errorf("Expected 'Q3AM3UQ867SPQQA43P2F', got %s'", credValues.AccessKeyID) } if credValues.SecretAccessKey != "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" { t.Errorf("Expected 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG', got %s'", credValues.SecretAccessKey) } if credValues.SignerType != SignatureV2 { t.Errorf("Expected 'S3v2', got %s'", credValues.SignerType) } os.Clearenv() creds = NewFileMinioClient("config.json.sample", "play") credValues, err = creds.GetWithContext(defaultCredContext) if err != nil { t.Fatal(err) } if credValues.AccessKeyID != "Q3AM3UQ867SPQQA43P2F" { t.Errorf("Expected 'Q3AM3UQ867SPQQA43P2F', got %s'", credValues.AccessKeyID) } if credValues.SecretAccessKey != "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" { t.Errorf("Expected 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG', got %s'", credValues.SecretAccessKey) } if credValues.SignerType != SignatureV2 { t.Errorf("Expected 'S3v2', got %s'", credValues.SignerType) } creds = NewFileMinioClient("non-existent.json", "play") _, err = creds.GetWithContext(defaultCredContext) if !os.IsNotExist(err) { t.Errorf("Expected open non-existent.json: no such file or directory, got %s", err) } if !creds.IsExpired() { t.Error("Should be expired if not loaded") } } minio-go-7.0.97/pkg/credentials/iam_aws.go000066400000000000000000000314321510244170000203560ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "bufio" "context" "encoding/json" "errors" "fmt" "io" "net" "net/http" "net/url" "os" "path" "strings" "time" ) // DefaultExpiryWindow - Default expiry window. // ExpiryWindow will allow the credentials to trigger refreshing // prior to the credentials actually expiring. This is beneficial // so race conditions with expiring credentials do not cause // request to fail unexpectedly due to ExpiredTokenException exceptions. // DefaultExpiryWindow can be used as parameter to (*Expiry).SetExpiration. // When used the tokens refresh will be triggered when 80% of the elapsed // time until the actual expiration time is passed. const DefaultExpiryWindow = -1 // A IAM retrieves credentials from the EC2 service, and keeps track if // those credentials are expired. type IAM struct { Expiry // Optional http Client to use when connecting to IAM metadata service // (overrides default client in CredContext) Client *http.Client // Custom endpoint to fetch IAM role credentials. Endpoint string // Region configurable custom region for STS Region string // Support for container authorization token https://docs.aws.amazon.com/sdkref/latest/guide/feature-container-credentials.html Container struct { AuthorizationToken string AuthorizationTokenFile string CredentialsFullURI string CredentialsRelativeURI string } // EKS based k8s RBAC authorization - https://docs.aws.amazon.com/eks/latest/userguide/pod-configuration.html EKSIdentity struct { TokenFile string RoleARN string RoleSessionName string } } // IAM Roles for Amazon EC2 // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html const ( DefaultIAMRoleEndpoint = "http://169.254.169.254" DefaultECSRoleEndpoint = "http://169.254.170.2" DefaultSTSRoleEndpoint = "https://sts.amazonaws.com" DefaultIAMSecurityCredsPath = "/latest/meta-data/iam/security-credentials/" TokenRequestTTLHeader = "X-aws-ec2-metadata-token-ttl-seconds" TokenPath = "/latest/api/token" TokenTTL = "21600" TokenRequestHeader = "X-aws-ec2-metadata-token" ) // NewIAM returns a pointer to a new Credentials object wrapping the IAM. func NewIAM(endpoint string) *Credentials { return New(&IAM{ Endpoint: endpoint, }) } // RetrieveWithCredContext is like Retrieve with Cred Context func (m *IAM) RetrieveWithCredContext(cc *CredContext) (Value, error) { if cc == nil { cc = defaultCredContext } token := os.Getenv("AWS_CONTAINER_AUTHORIZATION_TOKEN") if token == "" { token = m.Container.AuthorizationToken } tokenFile := os.Getenv("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE") if tokenFile == "" { tokenFile = m.Container.AuthorizationToken } relativeURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") if relativeURI == "" { relativeURI = m.Container.CredentialsRelativeURI } fullURI := os.Getenv("AWS_CONTAINER_CREDENTIALS_FULL_URI") if fullURI == "" { fullURI = m.Container.CredentialsFullURI } identityFile := os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE") if identityFile == "" { identityFile = m.EKSIdentity.TokenFile } roleArn := os.Getenv("AWS_ROLE_ARN") if roleArn == "" { roleArn = m.EKSIdentity.RoleARN } roleSessionName := os.Getenv("AWS_ROLE_SESSION_NAME") if roleSessionName == "" { roleSessionName = m.EKSIdentity.RoleSessionName } region := os.Getenv("AWS_REGION") if region == "" { region = m.Region } var roleCreds ec2RoleCredRespBody var err error client := m.Client if client == nil { client = cc.Client } if client == nil { client = defaultCredContext.Client } endpoint := m.Endpoint switch { case identityFile != "": if len(endpoint) == 0 { if region != "" { if strings.HasPrefix(region, "cn-") { endpoint = "https://sts." + region + ".amazonaws.com.cn" } else { endpoint = "https://sts." + region + ".amazonaws.com" } } else { endpoint = DefaultSTSRoleEndpoint } } creds := &STSWebIdentity{ Client: client, STSEndpoint: endpoint, GetWebIDTokenExpiry: func() (*WebIdentityToken, error) { token, err := os.ReadFile(identityFile) if err != nil { return nil, err } return &WebIdentityToken{Token: string(token)}, nil }, RoleARN: roleArn, roleSessionName: roleSessionName, } stsWebIdentityCreds, err := creds.RetrieveWithCredContext(cc) if err == nil { m.SetExpiration(creds.Expiration(), DefaultExpiryWindow) } return stsWebIdentityCreds, err case relativeURI != "": if len(endpoint) == 0 { endpoint = fmt.Sprintf("%s%s", DefaultECSRoleEndpoint, relativeURI) } roleCreds, err = getEcsTaskCredentials(client, endpoint, token) case tokenFile != "" && fullURI != "": endpoint = fullURI roleCreds, err = getEKSPodIdentityCredentials(client, endpoint, tokenFile) case fullURI != "": if len(endpoint) == 0 { endpoint = fullURI var ok bool if ok, err = isLoopback(endpoint); !ok { if err == nil { err = fmt.Errorf("uri host is not a loopback address: %s", endpoint) } break } } roleCreds, err = getEcsTaskCredentials(client, endpoint, token) default: roleCreds, err = getCredentials(client, endpoint) } if err != nil { return Value{}, err } // Expiry window is set to 10secs. m.SetExpiration(roleCreds.Expiration, DefaultExpiryWindow) return Value{ AccessKeyID: roleCreds.AccessKeyID, SecretAccessKey: roleCreds.SecretAccessKey, SessionToken: roleCreds.Token, Expiration: roleCreds.Expiration, SignerType: SignatureV4, }, nil } // Retrieve retrieves credentials from the EC2 service. // Error will be returned if the request fails, or unable to extract // the desired func (m *IAM) Retrieve() (Value, error) { return m.RetrieveWithCredContext(nil) } // A ec2RoleCredRespBody provides the shape for unmarshaling credential // request responses. type ec2RoleCredRespBody struct { // Success State Expiration time.Time AccessKeyID string SecretAccessKey string Token string // Error state Code string Message string // Unused params. LastUpdated time.Time Type string } // Get the final IAM role URL where the request will // be sent to fetch the rolling access credentials. // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html func getIAMRoleURL(endpoint string) (*url.URL, error) { u, err := url.Parse(endpoint) if err != nil { return nil, err } u.Path = DefaultIAMSecurityCredsPath return u, nil } // listRoleNames lists of credential role names associated // with the current EC2 service. If there are no credentials, // or there is an error making or receiving the request. // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html func listRoleNames(client *http.Client, u *url.URL, token string) ([]string, error) { req, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { return nil, err } if token != "" { req.Header.Add(TokenRequestHeader, token) } resp, err := client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, errors.New(resp.Status) } credsList := []string{} s := bufio.NewScanner(resp.Body) for s.Scan() { credsList = append(credsList, s.Text()) } if err := s.Err(); err != nil { return nil, err } return credsList, nil } func getEcsTaskCredentials(client *http.Client, endpoint, token string) (ec2RoleCredRespBody, error) { req, err := http.NewRequest(http.MethodGet, endpoint, nil) if err != nil { return ec2RoleCredRespBody{}, err } if token != "" { req.Header.Set("Authorization", token) } resp, err := client.Do(req) if err != nil { return ec2RoleCredRespBody{}, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return ec2RoleCredRespBody{}, errors.New(resp.Status) } respCreds := ec2RoleCredRespBody{} if err := json.NewDecoder(resp.Body).Decode(&respCreds); err != nil { return ec2RoleCredRespBody{}, err } return respCreds, nil } func getEKSPodIdentityCredentials(client *http.Client, endpoint string, tokenFile string) (ec2RoleCredRespBody, error) { if tokenFile != "" { bytes, err := os.ReadFile(tokenFile) if err != nil { return ec2RoleCredRespBody{}, fmt.Errorf("getEKSPodIdentityCredentials: failed to read token file:%s", err) } token := string(bytes) return getEcsTaskCredentials(client, endpoint, token) } return ec2RoleCredRespBody{}, fmt.Errorf("getEKSPodIdentityCredentials: no tokenFile found") } func fetchIMDSToken(client *http.Client, endpoint string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodPut, endpoint+TokenPath, nil) if err != nil { return "", err } req.Header.Add(TokenRequestTTLHeader, TokenTTL) resp, err := client.Do(req) if err != nil { return "", err } defer resp.Body.Close() data, err := io.ReadAll(resp.Body) if err != nil { return "", err } if resp.StatusCode != http.StatusOK { return "", errors.New(resp.Status) } return string(data), nil } // getCredentials - obtains the credentials from the IAM role name associated with // the current EC2 service. // // If the credentials cannot be found, or there is an error // reading the response an error will be returned. func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, error) { if endpoint == "" { endpoint = DefaultIAMRoleEndpoint } // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html token, err := fetchIMDSToken(client, endpoint) if err != nil { // Return only errors for valid situations, if the IMDSv2 is not enabled // we will not be able to get the token, in such a situation we have // to rely on IMDSv1 behavior as a fallback, this check ensures that. // Refer https://github.com/minio/minio-go/issues/1866 if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) { return ec2RoleCredRespBody{}, err } } // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html u, err := getIAMRoleURL(endpoint) if err != nil { return ec2RoleCredRespBody{}, err } // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html roleNames, err := listRoleNames(client, u, token) if err != nil { return ec2RoleCredRespBody{}, err } if len(roleNames) == 0 { return ec2RoleCredRespBody{}, errors.New("No IAM roles attached to this EC2 service") } // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html // - An instance profile can contain only one IAM role. This limit cannot be increased. roleName := roleNames[0] // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html // The following command retrieves the security credentials for an // IAM role named `s3access`. // // $ curl http://169.254.169.254/latest/meta-data/iam/security-credentials/s3access // u.Path = path.Join(u.Path, roleName) req, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { return ec2RoleCredRespBody{}, err } if token != "" { req.Header.Add(TokenRequestHeader, token) } resp, err := client.Do(req) if err != nil { return ec2RoleCredRespBody{}, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return ec2RoleCredRespBody{}, errors.New(resp.Status) } respCreds := ec2RoleCredRespBody{} if err := json.NewDecoder(resp.Body).Decode(&respCreds); err != nil { return ec2RoleCredRespBody{}, err } if respCreds.Code != "Success" { // If an error code was returned something failed requesting the role. return ec2RoleCredRespBody{}, errors.New(respCreds.Message) } return respCreds, nil } // isLoopback identifies if a uri's host is on a loopback address func isLoopback(uri string) (bool, error) { u, err := url.Parse(uri) if err != nil { return false, err } host := u.Hostname() if len(host) == 0 { return false, fmt.Errorf("can't parse host from uri: %s", uri) } ips, err := net.LookupHost(host) if err != nil { return false, err } for _, ip := range ips { if !net.ParseIP(ip).IsLoopback() { return false, nil } } return true, nil } minio-go-7.0.97/pkg/credentials/iam_aws_test.go000066400000000000000000000260161510244170000214170ustar00rootroot00000000000000//go:build !windows // +build !windows /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "fmt" "net/http" "net/http/httptest" "os" "strconv" "testing" "time" ) const credsRespTmpl = `{ "Code": "Success", "Type": "AWS-HMAC", "AccessKeyId" : "accessKey", "SecretAccessKey" : "secret", "Token" : "token", "Expiration" : "%s", "LastUpdated" : "2009-11-23T0:00:00Z" }` const credsFailRespTmpl = `{ "Code": "ErrorCode", "Message": "ErrorMsg", "LastUpdated": "2009-11-23T0:00:00Z" }` const credsRespEcsTaskTmpl = `{ "AccessKeyId" : "accessKey", "SecretAccessKey" : "secret", "Token" : "token", "Expiration" : "%s" }` const credsRespStsImpl = ` amzn1.account.AF6RHO7KZU5XRVQJGXK6HB56KR2A client.5498841531868486423.1548@apps.example.com arn:aws:sts::123456789012:assumed-role/FederatedWebIdentityRole/app1 AROACLKWSDQRAOEXAMPLE:app1 token secret %s accessKey www.amazon.com ad4156e9-bce1-11e2-82e6-6b6efEXAMPLE ` func initTestFailServer() *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "Not allowed", http.StatusBadRequest) })) return server } func initTestServerNoRoles() *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("")) })) return server } // Instance Metadata Service with V1 disabled. func initIMDSv2Server(expireOn string, failAssume bool) *httptest.Server { imdsToken := "IMDSTokenabc123==" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Println(r.URL.Path) fmt.Println(r.Method) if r.URL.Path == "/latest/api/token" && r.Method == "PUT" { ttlHeader := r.Header.Get("X-aws-ec2-metadata-token-ttl-seconds") ttl, err := strconv.ParseInt(ttlHeader, 10, 32) if err != nil || ttl < 0 || ttl > 21600 { http.Error(w, "", http.StatusBadRequest) return } w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", ttlHeader) w.Write([]byte(imdsToken)) return } token := r.Header.Get("X-aws-ec2-metadata-token") if token != imdsToken { http.Error(w, r.URL.Path, http.StatusUnauthorized) return } switch r.URL.Path { case "/latest/meta-data/iam/security-credentials/": fmt.Fprintln(w, "RoleName") case "/latest/meta-data/iam/security-credentials/RoleName": if failAssume { fmt.Fprint(w, credsFailRespTmpl) } else { fmt.Fprintf(w, credsRespTmpl, expireOn) } default: http.Error(w, "bad request", http.StatusBadRequest) } })) return server } func initEcsTaskTestServer(expireOn string) *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprintf(w, credsRespEcsTaskTmpl, expireOn) })) return server } func initStsTestServer(expireOn string) *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } required := []string{"RoleArn", "RoleSessionName", "WebIdentityToken", "Version"} for _, field := range required { if _, ok := r.Form[field]; !ok { http.Error(w, fmt.Sprintf("%s missing", field), http.StatusBadRequest) return } } fmt.Fprintf(w, credsRespStsImpl, expireOn) })) return server } func TestIAMMalformedEndpoint(t *testing.T) { creds := NewIAM("%%%%") _, err := creds.GetWithContext(defaultCredContext) if err == nil { t.Fatal("Unexpected should fail here") } } func TestIAMFailServer(t *testing.T) { server := initTestFailServer() defer server.Close() creds := NewIAM(server.URL) _, err := creds.GetWithContext(defaultCredContext) if err == nil { t.Fatal("Unexpected should fail here") } if err.Error() != "400 Bad Request" { t.Fatalf("Expected '400 Bad Request', got %s", err) } } func TestIAMNoRoles(t *testing.T) { server := initTestServerNoRoles() defer server.Close() creds := NewIAM(server.URL) _, err := creds.GetWithContext(defaultCredContext) if err == nil { t.Fatal("Unexpected should fail here") } if err.Error() != "No IAM roles attached to this EC2 service" { t.Fatalf("Expected 'No IAM roles attached to this EC2 service', got %s", err) } } func TestIAM(t *testing.T) { server := initIMDSv2Server("2014-12-16T01:51:37Z", false) defer server.Close() p := &IAM{ Endpoint: server.URL, } creds, err := p.RetrieveWithCredContext(defaultCredContext) if err != nil { t.Fatal(err) } if creds.AccessKeyID != "accessKey" { t.Errorf("Expected \"accessKey\", got %s", creds.AccessKeyID) } if creds.SecretAccessKey != "secret" { t.Errorf("Expected \"secret\", got %s", creds.SecretAccessKey) } if creds.SessionToken != "token" { t.Errorf("Expected \"token\", got %s", creds.SessionToken) } if !p.IsExpired() { t.Error("Expected creds to be expired.") } } func TestIAMFailAssume(t *testing.T) { server := initIMDSv2Server("2014-12-16T01:51:37Z", true) defer server.Close() p := &IAM{ Endpoint: server.URL, } _, err := p.RetrieveWithCredContext(defaultCredContext) if err == nil { t.Fatal("Unexpected success, should fail") } if err.Error() != "ErrorMsg" { t.Errorf("Expected \"ErrorMsg\", got %s", err) } } func TestIAMIsExpired(t *testing.T) { server := initIMDSv2Server("2014-12-16T01:51:37Z", false) defer server.Close() p := &IAM{ Endpoint: server.URL, } p.CurrentTime = func() time.Time { return time.Date(2014, 12, 15, 21, 26, 0, 0, time.UTC) } if !p.IsExpired() { t.Error("Expected creds to be expired before retrieve.") } _, err := p.RetrieveWithCredContext(defaultCredContext) if err != nil { t.Fatal(err) } if p.IsExpired() { t.Error("Expected creds to not be expired after retrieve.") } p.CurrentTime = func() time.Time { return time.Date(3014, 12, 15, 21, 26, 0, 0, time.UTC) } if !p.IsExpired() { t.Error("Expected creds to be expired when curren time has changed") } } func TestEcsTask(t *testing.T) { server := initEcsTaskTestServer("2014-12-16T01:51:37Z") defer server.Close() p := &IAM{ Endpoint: server.URL, } t.Setenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "/v2/credentials?id=task_credential_id") creds, err := p.RetrieveWithCredContext(defaultCredContext) os.Unsetenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") if err != nil { t.Errorf("Unexpected failure %s", err) } if creds.AccessKeyID != "accessKey" { t.Errorf("Expected \"accessKey\", got %s", creds.AccessKeyID) } if creds.SecretAccessKey != "secret" { t.Errorf("Expected \"secret\", got %s", creds.SecretAccessKey) } if creds.SessionToken != "token" { t.Errorf("Expected \"token\", got %s", creds.SessionToken) } if !p.IsExpired() { t.Error("Expected creds to be expired.") } } func TestEcsTaskFullURI(t *testing.T) { server := initEcsTaskTestServer("2014-12-16T01:51:37Z") defer server.Close() p := &IAM{} t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", fmt.Sprintf("%s%s", server.URL, "/v2/credentials?id=task_credential_id")) creds, err := p.RetrieveWithCredContext(defaultCredContext) os.Unsetenv("AWS_CONTAINER_CREDENTIALS_FULL_URI") if err != nil { t.Errorf("Unexpected failure %s", err) } if creds.AccessKeyID != "accessKey" { t.Errorf("Expected \"accessKey\", got %s", creds.AccessKeyID) } if creds.SecretAccessKey != "secret" { t.Errorf("Expected \"secret\", got %s", creds.SecretAccessKey) } if creds.SessionToken != "token" { t.Errorf("Expected \"token\", got %s", creds.SessionToken) } if !p.IsExpired() { t.Error("Expected creds to be expired.") } } func TestSts(t *testing.T) { server := initStsTestServer("2014-12-16T01:51:37Z") defer server.Close() p := &IAM{ Endpoint: server.URL, } f, err := os.CreateTemp(t.TempDir(), "minio-go") if err != nil { t.Errorf("Unexpected failure %s", err) } defer os.Remove(f.Name()) f.Write([]byte("token")) f.Close() t.Setenv("AWS_WEB_IDENTITY_TOKEN_FILE", f.Name()) t.Setenv("AWS_ROLE_ARN", "arn:aws:sts::123456789012:assumed-role/FederatedWebIdentityRole/app1") creds, err := p.RetrieveWithCredContext(defaultCredContext) os.Unsetenv("AWS_WEB_IDENTITY_TOKEN_FILE") os.Unsetenv("AWS_ROLE_ARN") if err != nil { t.Errorf("Unexpected failure %s", err) } if creds.AccessKeyID != "accessKey" { t.Errorf("Expected \"accessKey\", got %s", creds.AccessKeyID) } if creds.SecretAccessKey != "secret" { t.Errorf("Expected \"secret\", got %s", creds.SecretAccessKey) } if creds.SessionToken != "token" { t.Errorf("Expected \"token\", got %s", creds.SessionToken) } if !p.IsExpired() { t.Error("Expected creds to be expired.") } } func TestStsCn(t *testing.T) { server := initStsTestServer("2014-12-16T01:51:37Z") defer server.Close() p := &IAM{ Endpoint: server.URL, } f, err := os.CreateTemp(t.TempDir(), "minio-go") if err != nil { t.Errorf("Unexpected failure %s", err) } defer os.Remove(f.Name()) f.Write([]byte("token")) f.Close() t.Setenv("AWS_REGION", "cn-northwest-1") t.Setenv("AWS_WEB_IDENTITY_TOKEN_FILE", f.Name()) t.Setenv("AWS_ROLE_ARN", "arn:aws:sts::123456789012:assumed-role/FederatedWebIdentityRole/app1") creds, err := p.RetrieveWithCredContext(defaultCredContext) os.Unsetenv("AWS_WEB_IDENTITY_TOKEN_FILE") os.Unsetenv("AWS_ROLE_ARN") if err != nil { t.Errorf("Unexpected failure %s", err) } if creds.AccessKeyID != "accessKey" { t.Errorf("Expected \"accessKey\", got %s", creds.AccessKeyID) } if creds.SecretAccessKey != "secret" { t.Errorf("Expected \"secret\", got %s", creds.SecretAccessKey) } if creds.SessionToken != "token" { t.Errorf("Expected \"token\", got %s", creds.SessionToken) } if !p.IsExpired() { t.Error("Expected creds to be expired.") } } func TestIMDSv1Blocked(t *testing.T) { server := initIMDSv2Server("2014-12-16T01:51:37Z", false) p := &IAM{ Endpoint: server.URL, } _, err := p.RetrieveWithCredContext(defaultCredContext) if err != nil { t.Errorf("Unexpected IMDSv2 failure %s", err) } } minio-go-7.0.97/pkg/credentials/signature_type.go000066400000000000000000000041471510244170000220030ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import "strings" // SignatureType is type of Authorization requested for a given HTTP request. type SignatureType int // Different types of supported signatures - default is SignatureV4 or SignatureDefault. const ( // SignatureDefault is always set to v4. SignatureDefault SignatureType = iota SignatureV4 SignatureV2 SignatureV4Streaming SignatureAnonymous // Anonymous signature signifies, no signature. ) // IsV2 - is signature SignatureV2? func (s SignatureType) IsV2() bool { return s == SignatureV2 } // IsV4 - is signature SignatureV4? func (s SignatureType) IsV4() bool { return s == SignatureV4 || s == SignatureDefault } // IsStreamingV4 - is signature SignatureV4Streaming? func (s SignatureType) IsStreamingV4() bool { return s == SignatureV4Streaming } // IsAnonymous - is signature empty? func (s SignatureType) IsAnonymous() bool { return s == SignatureAnonymous } // Stringer humanized version of signature type, // strings returned here are case insensitive. func (s SignatureType) String() string { if s.IsV2() { return "S3v2" } else if s.IsV4() { return "S3v4" } else if s.IsStreamingV4() { return "S3v4Streaming" } return "Anonymous" } func parseSignatureType(str string) SignatureType { if strings.EqualFold(str, "S3v4") { return SignatureV4 } else if strings.EqualFold(str, "S3v2") { return SignatureV2 } else if strings.EqualFold(str, "S3v4Streaming") { return SignatureV4Streaming } return SignatureAnonymous } minio-go-7.0.97/pkg/credentials/static.go000066400000000000000000000043251510244170000202260ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials // A Static is a set of credentials which are set programmatically, // and will never expire. type Static struct { Value } // NewStaticV2 returns a pointer to a new Credentials object // wrapping a static credentials value provider, signature is // set to v2. If access and secret are not specified then // regardless of signature type set it Value will return // as anonymous. func NewStaticV2(id, secret, token string) *Credentials { return NewStatic(id, secret, token, SignatureV2) } // NewStaticV4 is similar to NewStaticV2 with similar considerations. func NewStaticV4(id, secret, token string) *Credentials { return NewStatic(id, secret, token, SignatureV4) } // NewStatic returns a pointer to a new Credentials object // wrapping a static credentials value provider. func NewStatic(id, secret, token string, signerType SignatureType) *Credentials { return New(&Static{ Value: Value{ AccessKeyID: id, SecretAccessKey: secret, SessionToken: token, SignerType: signerType, }, }) } // Retrieve returns the static credentials. func (s *Static) Retrieve() (Value, error) { if s.AccessKeyID == "" || s.SecretAccessKey == "" { // Anonymous is not an error return Value{SignerType: SignatureAnonymous}, nil } return s.Value, nil } // RetrieveWithCredContext returns the static credentials. func (s *Static) RetrieveWithCredContext(_ *CredContext) (Value, error) { return s.Retrieve() } // IsExpired returns if the credentials are expired. // // For Static, the credentials never expired. func (s *Static) IsExpired() bool { return false } minio-go-7.0.97/pkg/credentials/static_test.go000066400000000000000000000037671510244170000212760ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import "testing" func TestStaticGet(t *testing.T) { creds := NewStatic("UXHW", "SECRET", "", SignatureV4) credValues, err := creds.GetWithContext(defaultCredContext) if err != nil { t.Fatal(err) } if credValues.AccessKeyID != "UXHW" { t.Errorf("Expected access key ID to match \"UXHW\", got %s", credValues.AccessKeyID) } if credValues.SecretAccessKey != "SECRET" { t.Errorf("Expected secret access key to match \"SECRET\", got %s", credValues.SecretAccessKey) } if credValues.SessionToken != "" { t.Error("Expected session token to match") } if credValues.SignerType != SignatureV4 { t.Errorf("Expected 'S3v4', got %s", credValues.SignerType) } if creds.IsExpired() { t.Error("Static credentials should never expire") } creds = NewStatic("", "", "", SignatureDefault) credValues, err = creds.GetWithContext(defaultCredContext) if err != nil { t.Fatal(err) } if credValues.AccessKeyID != "" { t.Errorf("Expected access key ID to match empty string, got %s", credValues.AccessKeyID) } if credValues.SecretAccessKey != "" { t.Errorf("Expected secret access key to match empty string, got %s", credValues.SecretAccessKey) } if !credValues.SignerType.IsAnonymous() { t.Errorf("Expected 'Anonymous', got %s", credValues.SignerType) } if creds.IsExpired() { t.Error("Static credentials should never expire") } } minio-go-7.0.97/pkg/credentials/sts_client_grants.go000066400000000000000000000150261510244170000224640ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2019-2022 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "bytes" "encoding/xml" "errors" "fmt" "io" "net/http" "net/url" "strings" "time" ) // AssumedRoleUser - The identifiers for the temporary security credentials that // the operation returns. Please also see https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/AssumedRoleUser type AssumedRoleUser struct { Arn string AssumedRoleID string `xml:"AssumeRoleId"` } // AssumeRoleWithClientGrantsResponse contains the result of successful AssumeRoleWithClientGrants request. type AssumeRoleWithClientGrantsResponse struct { XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithClientGrantsResponse" json:"-"` Result ClientGrantsResult `xml:"AssumeRoleWithClientGrantsResult"` ResponseMetadata struct { RequestID string `xml:"RequestId,omitempty"` } `xml:"ResponseMetadata,omitempty"` } // ClientGrantsResult - Contains the response to a successful AssumeRoleWithClientGrants // request, including temporary credentials that can be used to make MinIO API requests. type ClientGrantsResult struct { AssumedRoleUser AssumedRoleUser `xml:",omitempty"` Audience string `xml:",omitempty"` Credentials struct { AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"` SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"` Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"` SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"` } `xml:",omitempty"` PackedPolicySize int `xml:",omitempty"` Provider string `xml:",omitempty"` SubjectFromClientGrantsToken string `xml:",omitempty"` } // ClientGrantsToken - client grants token with expiry. type ClientGrantsToken struct { Token string Expiry int } // A STSClientGrants retrieves credentials from MinIO service, and keeps track if // those credentials are expired. type STSClientGrants struct { Expiry // Optional http Client to use when connecting to MinIO STS service. // (overrides default client in CredContext) Client *http.Client // MinIO endpoint to fetch STS credentials. STSEndpoint string // getClientGrantsTokenExpiry function to retrieve tokens // from IDP This function should return two values one is // accessToken which is a self contained access token (JWT) // and second return value is the expiry associated with // this token. This is a customer provided function and // is mandatory. GetClientGrantsTokenExpiry func() (*ClientGrantsToken, error) } // NewSTSClientGrants returns a pointer to a new // Credentials object wrapping the STSClientGrants. func NewSTSClientGrants(stsEndpoint string, getClientGrantsTokenExpiry func() (*ClientGrantsToken, error)) (*Credentials, error) { if getClientGrantsTokenExpiry == nil { return nil, errors.New("Client grants access token and expiry retrieval function should be defined") } return New(&STSClientGrants{ STSEndpoint: stsEndpoint, GetClientGrantsTokenExpiry: getClientGrantsTokenExpiry, }), nil } func getClientGrantsCredentials(clnt *http.Client, endpoint string, getClientGrantsTokenExpiry func() (*ClientGrantsToken, error), ) (AssumeRoleWithClientGrantsResponse, error) { accessToken, err := getClientGrantsTokenExpiry() if err != nil { return AssumeRoleWithClientGrantsResponse{}, err } v := url.Values{} v.Set("Action", "AssumeRoleWithClientGrants") v.Set("Token", accessToken.Token) v.Set("DurationSeconds", fmt.Sprintf("%d", accessToken.Expiry)) v.Set("Version", STSVersion) u, err := url.Parse(endpoint) if err != nil { return AssumeRoleWithClientGrantsResponse{}, err } req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(v.Encode())) if err != nil { return AssumeRoleWithClientGrantsResponse{}, err } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") resp, err := clnt.Do(req) if err != nil { return AssumeRoleWithClientGrantsResponse{}, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { var errResp ErrorResponse buf, err := io.ReadAll(resp.Body) if err != nil { return AssumeRoleWithClientGrantsResponse{}, err } _, err = xmlDecodeAndBody(bytes.NewReader(buf), &errResp) if err != nil { var s3Err Error if _, err = xmlDecodeAndBody(bytes.NewReader(buf), &s3Err); err != nil { return AssumeRoleWithClientGrantsResponse{}, err } errResp.RequestID = s3Err.RequestID errResp.STSError.Code = s3Err.Code errResp.STSError.Message = s3Err.Message } return AssumeRoleWithClientGrantsResponse{}, errResp } a := AssumeRoleWithClientGrantsResponse{} if err = xml.NewDecoder(resp.Body).Decode(&a); err != nil { return AssumeRoleWithClientGrantsResponse{}, err } return a, nil } // RetrieveWithCredContext is like Retrieve() with cred context func (m *STSClientGrants) RetrieveWithCredContext(cc *CredContext) (Value, error) { if cc == nil { cc = defaultCredContext } client := m.Client if client == nil { client = cc.Client } if client == nil { client = defaultCredContext.Client } stsEndpoint := m.STSEndpoint if stsEndpoint == "" { stsEndpoint = cc.Endpoint } if stsEndpoint == "" { return Value{}, errors.New("STS endpoint unknown") } a, err := getClientGrantsCredentials(client, stsEndpoint, m.GetClientGrantsTokenExpiry) if err != nil { return Value{}, err } // Expiry window is set to 10secs. m.SetExpiration(a.Result.Credentials.Expiration, DefaultExpiryWindow) return Value{ AccessKeyID: a.Result.Credentials.AccessKey, SecretAccessKey: a.Result.Credentials.SecretKey, SessionToken: a.Result.Credentials.SessionToken, Expiration: a.Result.Credentials.Expiration, SignerType: SignatureV4, }, nil } // Retrieve retrieves credentials from the MinIO service. // Error will be returned if the request fails. func (m *STSClientGrants) Retrieve() (Value, error) { return m.RetrieveWithCredContext(nil) } minio-go-7.0.97/pkg/credentials/sts_custom_identity.go000066400000000000000000000115641510244170000230560ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2022 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "encoding/xml" "errors" "fmt" "net/http" "net/url" "time" ) // CustomTokenResult - Contains temporary creds and user metadata. type CustomTokenResult struct { Credentials struct { AccessKey string `xml:"AccessKeyId"` SecretKey string `xml:"SecretAccessKey"` Expiration time.Time `xml:"Expiration"` SessionToken string `xml:"SessionToken"` } `xml:",omitempty"` AssumedUser string `xml:",omitempty"` } // AssumeRoleWithCustomTokenResponse contains the result of a successful // AssumeRoleWithCustomToken request. type AssumeRoleWithCustomTokenResponse struct { XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithCustomTokenResponse" json:"-"` Result CustomTokenResult `xml:"AssumeRoleWithCustomTokenResult"` Metadata struct { RequestID string `xml:"RequestId,omitempty"` } `xml:"ResponseMetadata,omitempty"` } // CustomTokenIdentity - satisfies the Provider interface, and retrieves // credentials from MinIO using the AssumeRoleWithCustomToken STS API. type CustomTokenIdentity struct { Expiry // Optional http Client to use when connecting to MinIO STS service. // (overrides default client in CredContext) Client *http.Client // MinIO server STS endpoint to fetch STS credentials. STSEndpoint string // The custom token to use with the request. Token string // RoleArn associated with the identity RoleArn string // RequestedExpiry is to set the validity of the generated credentials // (this value bounded by server). RequestedExpiry time.Duration // Optional, used for token revokation TokenRevokeType string } // RetrieveWithCredContext with Retrieve optionally cred context func (c *CustomTokenIdentity) RetrieveWithCredContext(cc *CredContext) (value Value, err error) { if cc == nil { cc = defaultCredContext } stsEndpoint := c.STSEndpoint if stsEndpoint == "" { stsEndpoint = cc.Endpoint } if stsEndpoint == "" { return Value{}, errors.New("STS endpoint unknown") } u, err := url.Parse(stsEndpoint) if err != nil { return value, err } v := url.Values{} v.Set("Action", "AssumeRoleWithCustomToken") v.Set("Version", STSVersion) v.Set("RoleArn", c.RoleArn) v.Set("Token", c.Token) if c.RequestedExpiry != 0 { v.Set("DurationSeconds", fmt.Sprintf("%d", int(c.RequestedExpiry.Seconds()))) } if c.TokenRevokeType != "" { v.Set("TokenRevokeType", c.TokenRevokeType) } u.RawQuery = v.Encode() req, err := http.NewRequest(http.MethodPost, u.String(), nil) if err != nil { return value, err } client := c.Client if client == nil { client = cc.Client } if client == nil { client = defaultCredContext.Client } resp, err := client.Do(req) if err != nil { return value, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return value, errors.New(resp.Status) } r := AssumeRoleWithCustomTokenResponse{} if err = xml.NewDecoder(resp.Body).Decode(&r); err != nil { return value, err } cr := r.Result.Credentials c.SetExpiration(cr.Expiration, DefaultExpiryWindow) return Value{ AccessKeyID: cr.AccessKey, SecretAccessKey: cr.SecretKey, SessionToken: cr.SessionToken, Expiration: cr.Expiration, SignerType: SignatureV4, }, nil } // Retrieve - to satisfy Provider interface; fetches credentials from MinIO. func (c *CustomTokenIdentity) Retrieve() (value Value, err error) { return c.RetrieveWithCredContext(nil) } // NewCustomTokenCredentials - returns credentials using the // AssumeRoleWithCustomToken STS API. func NewCustomTokenCredentials(stsEndpoint, token, roleArn string, optFuncs ...CustomTokenOpt) (*Credentials, error) { c := CustomTokenIdentity{ STSEndpoint: stsEndpoint, Token: token, RoleArn: roleArn, } for _, optFunc := range optFuncs { optFunc(&c) } return New(&c), nil } // CustomTokenOpt is a function type to configure the custom-token based // credentials using NewCustomTokenCredentials. type CustomTokenOpt func(*CustomTokenIdentity) // CustomTokenValidityOpt sets the validity duration of the requested // credentials. This value is ignored if the server enforces a lower validity // period. func CustomTokenValidityOpt(d time.Duration) CustomTokenOpt { return func(c *CustomTokenIdentity) { c.RequestedExpiry = d } } minio-go-7.0.97/pkg/credentials/sts_ldap_identity.go000066400000000000000000000151461510244170000224640ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2019-2022 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "bytes" "encoding/xml" "errors" "fmt" "io" "net/http" "net/url" "strings" "time" ) // AssumeRoleWithLDAPResponse contains the result of successful // AssumeRoleWithLDAPIdentity request type AssumeRoleWithLDAPResponse struct { XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithLDAPIdentityResponse" json:"-"` Result LDAPIdentityResult `xml:"AssumeRoleWithLDAPIdentityResult"` ResponseMetadata struct { RequestID string `xml:"RequestId,omitempty"` } `xml:"ResponseMetadata,omitempty"` } // LDAPIdentityResult - contains credentials for a successful // AssumeRoleWithLDAPIdentity request. type LDAPIdentityResult struct { Credentials struct { AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"` SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"` Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"` SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"` } `xml:",omitempty"` SubjectFromToken string `xml:",omitempty"` } // LDAPIdentity retrieves credentials from MinIO type LDAPIdentity struct { Expiry // Optional http Client to use when connecting to MinIO STS service. // (overrides default client in CredContext) Client *http.Client // Exported STS endpoint to fetch STS credentials. STSEndpoint string // LDAP username/password used to fetch LDAP STS credentials. LDAPUsername, LDAPPassword string // Session policy to apply to the generated credentials. Leave empty to // use the full access policy available to the user. Policy string // RequestedExpiry is the configured expiry duration for credentials // requested from LDAP. RequestedExpiry time.Duration // Optional, if empty applies to default config ConfigName string // Optional, used for token revokation TokenRevokeType string } // NewLDAPIdentity returns new credentials object that uses LDAP // Identity. func NewLDAPIdentity(stsEndpoint, ldapUsername, ldapPassword string, optFuncs ...LDAPIdentityOpt) (*Credentials, error) { l := LDAPIdentity{ STSEndpoint: stsEndpoint, LDAPUsername: ldapUsername, LDAPPassword: ldapPassword, } for _, optFunc := range optFuncs { optFunc(&l) } return New(&l), nil } // LDAPIdentityOpt is a function type used to configured the LDAPIdentity // instance. type LDAPIdentityOpt func(*LDAPIdentity) // LDAPIdentityPolicyOpt sets the session policy for requested credentials. func LDAPIdentityPolicyOpt(policy string) LDAPIdentityOpt { return func(k *LDAPIdentity) { k.Policy = policy } } // LDAPIdentityExpiryOpt sets the expiry duration for requested credentials. func LDAPIdentityExpiryOpt(d time.Duration) LDAPIdentityOpt { return func(k *LDAPIdentity) { k.RequestedExpiry = d } } // LDAPIdentityConfigNameOpt sets the config name for requested credentials. func LDAPIdentityConfigNameOpt(name string) LDAPIdentityOpt { return func(k *LDAPIdentity) { k.ConfigName = name } } // NewLDAPIdentityWithSessionPolicy returns new credentials object that uses // LDAP Identity with a specified session policy. The `policy` parameter must be // a JSON string specifying the policy document. // // Deprecated: Use the `LDAPIdentityPolicyOpt` with `NewLDAPIdentity` instead. func NewLDAPIdentityWithSessionPolicy(stsEndpoint, ldapUsername, ldapPassword, policy string) (*Credentials, error) { return New(&LDAPIdentity{ STSEndpoint: stsEndpoint, LDAPUsername: ldapUsername, LDAPPassword: ldapPassword, Policy: policy, }), nil } // RetrieveWithCredContext gets the credential by calling the MinIO STS API for // LDAP on the configured stsEndpoint. func (k *LDAPIdentity) RetrieveWithCredContext(cc *CredContext) (value Value, err error) { if cc == nil { cc = defaultCredContext } stsEndpoint := k.STSEndpoint if stsEndpoint == "" { stsEndpoint = cc.Endpoint } if stsEndpoint == "" { return Value{}, errors.New("STS endpoint unknown") } u, err := url.Parse(stsEndpoint) if err != nil { return value, err } v := url.Values{} v.Set("Action", "AssumeRoleWithLDAPIdentity") v.Set("Version", STSVersion) v.Set("LDAPUsername", k.LDAPUsername) v.Set("LDAPPassword", k.LDAPPassword) if k.Policy != "" { v.Set("Policy", k.Policy) } if k.RequestedExpiry != 0 { v.Set("DurationSeconds", fmt.Sprintf("%d", int(k.RequestedExpiry.Seconds()))) } if k.TokenRevokeType != "" { v.Set("TokenRevokeType", k.TokenRevokeType) } if k.ConfigName != "" { v.Set("ConfigName", k.ConfigName) } req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(v.Encode())) if err != nil { return value, err } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") client := k.Client if client == nil { client = cc.Client } if client == nil { client = defaultCredContext.Client } resp, err := client.Do(req) if err != nil { return value, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { var errResp ErrorResponse buf, err := io.ReadAll(resp.Body) if err != nil { return value, err } _, err = xmlDecodeAndBody(bytes.NewReader(buf), &errResp) if err != nil { var s3Err Error if _, err = xmlDecodeAndBody(bytes.NewReader(buf), &s3Err); err != nil { return value, err } errResp.RequestID = s3Err.RequestID errResp.STSError.Code = s3Err.Code errResp.STSError.Message = s3Err.Message } return value, errResp } r := AssumeRoleWithLDAPResponse{} if err = xml.NewDecoder(resp.Body).Decode(&r); err != nil { return value, err } cr := r.Result.Credentials k.SetExpiration(cr.Expiration, DefaultExpiryWindow) return Value{ AccessKeyID: cr.AccessKey, SecretAccessKey: cr.SecretKey, SessionToken: cr.SessionToken, Expiration: cr.Expiration, SignerType: SignatureV4, }, nil } // Retrieve gets the credential by calling the MinIO STS API for // LDAP on the configured stsEndpoint. func (k *LDAPIdentity) Retrieve() (value Value, err error) { return k.RetrieveWithCredContext(defaultCredContext) } minio-go-7.0.97/pkg/credentials/sts_tls_identity.go000066400000000000000000000161741510244170000223500ustar00rootroot00000000000000// MinIO Go Library for Amazon S3 Compatible Cloud Storage // Copyright 2021 MinIO, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package credentials import ( "bytes" "crypto/tls" "encoding/xml" "errors" "fmt" "io" "net/http" "net/url" "strconv" "time" ) // CertificateIdentityOption is an optional AssumeRoleWithCertificate // parameter - e.g. a custom HTTP transport configuration or S3 credental // livetime. type CertificateIdentityOption func(*STSCertificateIdentity) // CertificateIdentityWithTransport returns a CertificateIdentityOption that // customizes the STSCertificateIdentity with the given http.RoundTripper. func CertificateIdentityWithTransport(t http.RoundTripper) CertificateIdentityOption { return CertificateIdentityOption(func(i *STSCertificateIdentity) { if i.Client == nil { i.Client = &http.Client{} } i.Client.Transport = t }) } // CertificateIdentityWithExpiry returns a CertificateIdentityOption that // customizes the STSCertificateIdentity with the given livetime. // // Fetched S3 credentials will have the given livetime if the STS server // allows such credentials. func CertificateIdentityWithExpiry(livetime time.Duration) CertificateIdentityOption { return CertificateIdentityOption(func(i *STSCertificateIdentity) { i.S3CredentialLivetime = livetime }) } // A STSCertificateIdentity retrieves S3 credentials from the MinIO STS API and // rotates those credentials once they expire. type STSCertificateIdentity struct { Expiry // Optional http Client to use when connecting to MinIO STS service. // (overrides default client in CredContext) Client *http.Client // STSEndpoint is the base URL endpoint of the STS API. // For example, https://minio.local:9000 STSEndpoint string // S3CredentialLivetime is the duration temp. S3 access // credentials should be valid. // // It represents the access credential livetime requested // by the client. The STS server may choose to issue // temp. S3 credentials that have a different - usually // shorter - livetime. // // The default livetime is one hour. S3CredentialLivetime time.Duration // Certificate is the client certificate that is used for // STS authentication. Certificate tls.Certificate // Optional, used for token revokation TokenRevokeType string } // NewSTSCertificateIdentity returns a STSCertificateIdentity that authenticates // to the given STS endpoint with the given TLS certificate and retrieves and // rotates S3 credentials. func NewSTSCertificateIdentity(endpoint string, certificate tls.Certificate, options ...CertificateIdentityOption) (*Credentials, error) { identity := &STSCertificateIdentity{ STSEndpoint: endpoint, Certificate: certificate, } for _, option := range options { option(identity) } return New(identity), nil } // RetrieveWithCredContext is Retrieve with cred context func (i *STSCertificateIdentity) RetrieveWithCredContext(cc *CredContext) (Value, error) { if cc == nil { cc = defaultCredContext } stsEndpoint := i.STSEndpoint if stsEndpoint == "" { stsEndpoint = cc.Endpoint } if stsEndpoint == "" { return Value{}, errors.New("STS endpoint unknown") } endpointURL, err := url.Parse(stsEndpoint) if err != nil { return Value{}, err } livetime := i.S3CredentialLivetime if livetime == 0 { livetime = 1 * time.Hour } queryValues := url.Values{} queryValues.Set("Action", "AssumeRoleWithCertificate") queryValues.Set("Version", STSVersion) queryValues.Set("DurationSeconds", strconv.FormatUint(uint64(livetime.Seconds()), 10)) if i.TokenRevokeType != "" { queryValues.Set("TokenRevokeType", i.TokenRevokeType) } endpointURL.RawQuery = queryValues.Encode() req, err := http.NewRequest(http.MethodPost, endpointURL.String(), nil) if err != nil { return Value{}, err } client := i.Client if client == nil { client = cc.Client } if client == nil { client = defaultCredContext.Client } tr, ok := client.Transport.(*http.Transport) if !ok { return Value{}, fmt.Errorf("CredContext should contain an http.Transport value") } // Clone the HTTP transport (patch the TLS client certificate) trCopy := tr.Clone() trCopy.TLSClientConfig.Certificates = []tls.Certificate{i.Certificate} // Clone the HTTP client (patch the HTTP transport) clientCopy := *client clientCopy.Transport = trCopy resp, err := clientCopy.Do(req) if err != nil { return Value{}, err } if resp.Body != nil { defer resp.Body.Close() } if resp.StatusCode != http.StatusOK { var errResp ErrorResponse buf, err := io.ReadAll(resp.Body) if err != nil { return Value{}, err } _, err = xmlDecodeAndBody(bytes.NewReader(buf), &errResp) if err != nil { var s3Err Error if _, err = xmlDecodeAndBody(bytes.NewReader(buf), &s3Err); err != nil { return Value{}, err } errResp.RequestID = s3Err.RequestID errResp.STSError.Code = s3Err.Code errResp.STSError.Message = s3Err.Message } return Value{}, errResp } const MaxSize = 10 * 1 << 20 var body io.Reader = resp.Body if resp.ContentLength > 0 && resp.ContentLength < MaxSize { body = io.LimitReader(body, resp.ContentLength) } else { body = io.LimitReader(body, MaxSize) } var response assumeRoleWithCertificateResponse if err = xml.NewDecoder(body).Decode(&response); err != nil { return Value{}, err } i.SetExpiration(response.Result.Credentials.Expiration, DefaultExpiryWindow) return Value{ AccessKeyID: response.Result.Credentials.AccessKey, SecretAccessKey: response.Result.Credentials.SecretKey, SessionToken: response.Result.Credentials.SessionToken, Expiration: response.Result.Credentials.Expiration, SignerType: SignatureDefault, }, nil } // Retrieve fetches a new set of S3 credentials from the configured STS API endpoint. func (i *STSCertificateIdentity) Retrieve() (Value, error) { return i.RetrieveWithCredContext(defaultCredContext) } // Expiration returns the expiration time of the current S3 credentials. func (i *STSCertificateIdentity) Expiration() time.Time { return i.expiration } type assumeRoleWithCertificateResponse struct { XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithCertificateResponse" json:"-"` Result struct { Credentials struct { AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"` SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"` Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"` SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"` } `xml:"Credentials" json:"credentials,omitempty"` } `xml:"AssumeRoleWithCertificateResult"` ResponseMetadata struct { RequestID string `xml:"RequestId,omitempty"` } `xml:"ResponseMetadata,omitempty"` } minio-go-7.0.97/pkg/credentials/sts_web_identity.go000066400000000000000000000203551510244170000223170ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2019-2022 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package credentials import ( "bytes" "encoding/xml" "errors" "fmt" "io" "net/http" "net/url" "os" "strconv" "strings" "time" ) // AssumeRoleWithWebIdentityResponse contains the result of successful AssumeRoleWithWebIdentity request. type AssumeRoleWithWebIdentityResponse struct { XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithWebIdentityResponse" json:"-"` Result WebIdentityResult `xml:"AssumeRoleWithWebIdentityResult"` ResponseMetadata struct { RequestID string `xml:"RequestId,omitempty"` } `xml:"ResponseMetadata,omitempty"` } // WebIdentityResult - Contains the response to a successful AssumeRoleWithWebIdentity // request, including temporary credentials that can be used to make MinIO API requests. type WebIdentityResult struct { AssumedRoleUser AssumedRoleUser `xml:",omitempty"` Audience string `xml:",omitempty"` Credentials struct { AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty"` SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty"` Expiration time.Time `xml:"Expiration" json:"expiration,omitempty"` SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty"` } `xml:",omitempty"` PackedPolicySize int `xml:",omitempty"` Provider string `xml:",omitempty"` SubjectFromWebIdentityToken string `xml:",omitempty"` } // WebIdentityToken - web identity token with expiry. type WebIdentityToken struct { Token string AccessToken string RefreshToken string Expiry int } // A STSWebIdentity retrieves credentials from MinIO service, and keeps track if // those credentials are expired. type STSWebIdentity struct { Expiry // Optional http Client to use when connecting to MinIO STS service. // (overrides default client in CredContext) Client *http.Client // Exported STS endpoint to fetch STS credentials. STSEndpoint string // Exported GetWebIDTokenExpiry function which returns ID // tokens from IDP. This function should return two values // one is ID token which is a self contained ID token (JWT) // and second return value is the expiry associated with // this token. // This is a customer provided function and is mandatory. GetWebIDTokenExpiry func() (*WebIdentityToken, error) // RoleARN is the Amazon Resource Name (ARN) of the role that the caller is // assuming. RoleARN string // Policy is the policy where the credentials should be limited too. Policy string // roleSessionName is the identifier for the assumed role session. roleSessionName string // Optional, used for token revokation TokenRevokeType string } // NewSTSWebIdentity returns a pointer to a new // Credentials object wrapping the STSWebIdentity. func NewSTSWebIdentity(stsEndpoint string, getWebIDTokenExpiry func() (*WebIdentityToken, error), opts ...func(*STSWebIdentity)) (*Credentials, error) { if getWebIDTokenExpiry == nil { return nil, errors.New("Web ID token and expiry retrieval function should be defined") } i := &STSWebIdentity{ STSEndpoint: stsEndpoint, GetWebIDTokenExpiry: getWebIDTokenExpiry, } for _, o := range opts { o(i) } return New(i), nil } // NewKubernetesIdentity returns a pointer to a new // Credentials object using the Kubernetes service account func NewKubernetesIdentity(stsEndpoint string, opts ...func(*STSWebIdentity)) (*Credentials, error) { return NewSTSWebIdentity(stsEndpoint, func() (*WebIdentityToken, error) { token, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token") if err != nil { return nil, err } return &WebIdentityToken{ Token: string(token), }, nil }, opts...) } // WithPolicy option will enforce that the returned credentials // will be scoped down to the specified policy func WithPolicy(policy string) func(*STSWebIdentity) { return func(i *STSWebIdentity) { i.Policy = policy } } func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSessionName string, policy string, getWebIDTokenExpiry func() (*WebIdentityToken, error), tokenRevokeType string, ) (AssumeRoleWithWebIdentityResponse, error) { idToken, err := getWebIDTokenExpiry() if err != nil { return AssumeRoleWithWebIdentityResponse{}, err } v := url.Values{} v.Set("Action", "AssumeRoleWithWebIdentity") if len(roleARN) > 0 { v.Set("RoleArn", roleARN) if len(roleSessionName) == 0 { roleSessionName = strconv.FormatInt(time.Now().UnixNano(), 10) } v.Set("RoleSessionName", roleSessionName) } v.Set("WebIdentityToken", idToken.Token) if idToken.AccessToken != "" { // Usually set when server is using extended userInfo endpoint. v.Set("WebIdentityAccessToken", idToken.AccessToken) } if idToken.RefreshToken != "" { // Usually set when server is using extended userInfo endpoint. v.Set("WebIdentityRefreshToken", idToken.RefreshToken) } if idToken.Expiry > 0 { v.Set("DurationSeconds", fmt.Sprintf("%d", idToken.Expiry)) } if policy != "" { v.Set("Policy", policy) } v.Set("Version", STSVersion) if tokenRevokeType != "" { v.Set("TokenRevokeType", tokenRevokeType) } u, err := url.Parse(endpoint) if err != nil { return AssumeRoleWithWebIdentityResponse{}, err } req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(v.Encode())) if err != nil { return AssumeRoleWithWebIdentityResponse{}, err } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") resp, err := clnt.Do(req) if err != nil { return AssumeRoleWithWebIdentityResponse{}, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { var errResp ErrorResponse buf, err := io.ReadAll(resp.Body) if err != nil { return AssumeRoleWithWebIdentityResponse{}, err } _, err = xmlDecodeAndBody(bytes.NewReader(buf), &errResp) if err != nil { var s3Err Error if _, err = xmlDecodeAndBody(bytes.NewReader(buf), &s3Err); err != nil { return AssumeRoleWithWebIdentityResponse{}, err } errResp.RequestID = s3Err.RequestID errResp.STSError.Code = s3Err.Code errResp.STSError.Message = s3Err.Message } return AssumeRoleWithWebIdentityResponse{}, errResp } a := AssumeRoleWithWebIdentityResponse{} if err = xml.NewDecoder(resp.Body).Decode(&a); err != nil { return AssumeRoleWithWebIdentityResponse{}, err } return a, nil } // RetrieveWithCredContext is like Retrieve with optional cred context. func (m *STSWebIdentity) RetrieveWithCredContext(cc *CredContext) (Value, error) { if cc == nil { cc = defaultCredContext } client := m.Client if client == nil { client = cc.Client } if client == nil { client = defaultCredContext.Client } stsEndpoint := m.STSEndpoint if stsEndpoint == "" { stsEndpoint = cc.Endpoint } if stsEndpoint == "" { return Value{}, errors.New("STS endpoint unknown") } a, err := getWebIdentityCredentials(client, stsEndpoint, m.RoleARN, m.roleSessionName, m.Policy, m.GetWebIDTokenExpiry, m.TokenRevokeType) if err != nil { return Value{}, err } // Expiry window is set to 10secs. m.SetExpiration(a.Result.Credentials.Expiration, DefaultExpiryWindow) return Value{ AccessKeyID: a.Result.Credentials.AccessKey, SecretAccessKey: a.Result.Credentials.SecretKey, SessionToken: a.Result.Credentials.SessionToken, Expiration: a.Result.Credentials.Expiration, SignerType: SignatureV4, }, nil } // Retrieve retrieves credentials from the MinIO service. // Error will be returned if the request fails. func (m *STSWebIdentity) Retrieve() (Value, error) { return m.RetrieveWithCredContext(nil) } // Expiration returns the expiration time of the credentials func (m *STSWebIdentity) Expiration() time.Time { return m.expiration } minio-go-7.0.97/pkg/encrypt/000077500000000000000000000000001510244170000155735ustar00rootroot00000000000000minio-go-7.0.97/pkg/encrypt/fips_disabled.go000066400000000000000000000014101510244170000207060ustar00rootroot00000000000000//go:build !fips // +build !fips /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2022 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package encrypt // FIPS is true if 'fips' build tag was specified. const FIPS = false minio-go-7.0.97/pkg/encrypt/fips_enabled.go000066400000000000000000000014051510244170000205350ustar00rootroot00000000000000//go:build fips // +build fips /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2022 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package encrypt // FIPS is true if 'fips' build tag was specified. const FIPS = true minio-go-7.0.97/pkg/encrypt/server-side.go000066400000000000000000000143701510244170000203570ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package encrypt import ( "crypto/md5" "encoding/base64" "encoding/json" "errors" "net/http" "golang.org/x/crypto/argon2" ) const ( // SseGenericHeader is the AWS SSE header used for SSE-S3 and SSE-KMS. SseGenericHeader = "X-Amz-Server-Side-Encryption" // SseKmsKeyID is the AWS SSE-KMS key id. SseKmsKeyID = SseGenericHeader + "-Aws-Kms-Key-Id" // SseEncryptionContext is the AWS SSE-KMS Encryption Context data. SseEncryptionContext = SseGenericHeader + "-Context" // SseCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key. SseCustomerAlgorithm = SseGenericHeader + "-Customer-Algorithm" // SseCustomerKey is the AWS SSE-C encryption key HTTP header key. SseCustomerKey = SseGenericHeader + "-Customer-Key" // SseCustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key. SseCustomerKeyMD5 = SseGenericHeader + "-Customer-Key-MD5" // SseCopyCustomerAlgorithm is the AWS SSE-C algorithm HTTP header key for CopyObject API. SseCopyCustomerAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm" // SseCopyCustomerKey is the AWS SSE-C encryption key HTTP header key for CopyObject API. SseCopyCustomerKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key" // SseCopyCustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key for CopyObject API. SseCopyCustomerKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-MD5" ) // PBKDF creates a SSE-C key from the provided password and salt. // PBKDF is a password-based key derivation function // which can be used to derive a high-entropy cryptographic // key from a low-entropy password and a salt. type PBKDF func(password, salt []byte) ServerSide // DefaultPBKDF is the default PBKDF. It uses Argon2id with the // recommended parameters from the RFC draft (1 pass, 64 MB memory, 4 threads). var DefaultPBKDF PBKDF = func(password, salt []byte) ServerSide { sse := ssec{} copy(sse[:], argon2.IDKey(password, salt, 1, 64*1024, 4, 32)) return sse } // Type is the server-side-encryption method. It represents one of // the following encryption methods: // - SSE-C: server-side-encryption with customer provided keys // - KMS: server-side-encryption with managed keys // - S3: server-side-encryption using S3 storage encryption type Type string const ( // SSEC represents server-side-encryption with customer provided keys SSEC Type = "SSE-C" // KMS represents server-side-encryption with managed keys KMS Type = "KMS" // S3 represents server-side-encryption using S3 storage encryption S3 Type = "S3" ) // ServerSide is a form of S3 server-side-encryption. type ServerSide interface { // Type returns the server-side-encryption method. Type() Type // Marshal adds encryption headers to the provided HTTP headers. // It marks an HTTP request as server-side-encryption request // and inserts the required data into the headers. Marshal(h http.Header) } // NewSSE returns a server-side-encryption using S3 storage encryption. // Using SSE-S3 the server will encrypt the object with server-managed keys. func NewSSE() ServerSide { return s3{} } // NewSSEKMS returns a new server-side-encryption using SSE-KMS and the provided Key Id and context. func NewSSEKMS(keyID string, context interface{}) (ServerSide, error) { if context == nil { return kms{key: keyID, hasContext: false}, nil } serializedContext, err := json.Marshal(context) if err != nil { return nil, err } return kms{key: keyID, context: serializedContext, hasContext: true}, nil } // NewSSEC returns a new server-side-encryption using SSE-C and the provided key. // The key must be 32 bytes long. func NewSSEC(key []byte) (ServerSide, error) { if len(key) != 32 { return nil, errors.New("encrypt: SSE-C key must be 256 bit long") } sse := ssec{} copy(sse[:], key) return sse, nil } // SSE transforms a SSE-C copy encryption into a SSE-C encryption. // It is the inverse of SSECopy(...). // // If the provided sse is no SSE-C copy encryption SSE returns // sse unmodified. func SSE(sse ServerSide) ServerSide { if sse == nil || sse.Type() != SSEC { return sse } if sse, ok := sse.(ssecCopy); ok { return ssec(sse) } return sse } // SSECopy transforms a SSE-C encryption into a SSE-C copy // encryption. This is required for SSE-C key rotation or a SSE-C // copy where the source and the destination should be encrypted. // // If the provided sse is no SSE-C encryption SSECopy returns // sse unmodified. func SSECopy(sse ServerSide) ServerSide { if sse == nil || sse.Type() != SSEC { return sse } if sse, ok := sse.(ssec); ok { return ssecCopy(sse) } return sse } type ssec [32]byte func (s ssec) Type() Type { return SSEC } func (s ssec) Marshal(h http.Header) { keyMD5 := md5.Sum(s[:]) h.Set(SseCustomerAlgorithm, "AES256") h.Set(SseCustomerKey, base64.StdEncoding.EncodeToString(s[:])) h.Set(SseCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMD5[:])) } type ssecCopy [32]byte func (s ssecCopy) Type() Type { return SSEC } func (s ssecCopy) Marshal(h http.Header) { keyMD5 := md5.Sum(s[:]) h.Set(SseCopyCustomerAlgorithm, "AES256") h.Set(SseCopyCustomerKey, base64.StdEncoding.EncodeToString(s[:])) h.Set(SseCopyCustomerKeyMD5, base64.StdEncoding.EncodeToString(keyMD5[:])) } type s3 struct{} func (s s3) Type() Type { return S3 } func (s s3) Marshal(h http.Header) { h.Set(SseGenericHeader, "AES256") } type kms struct { key string context []byte hasContext bool } func (s kms) Type() Type { return KMS } func (s kms) Marshal(h http.Header) { h.Set(SseGenericHeader, "aws:kms") if s.key != "" { h.Set(SseKmsKeyID, s.key) } if s.hasContext { h.Set(SseEncryptionContext, base64.StdEncoding.EncodeToString(s.context)) } } minio-go-7.0.97/pkg/kvcache/000077500000000000000000000000001510244170000155135ustar00rootroot00000000000000minio-go-7.0.97/pkg/kvcache/cache.go000066400000000000000000000026101510244170000171040ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2025 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package kvcache import "sync" // Cache - Provides simple mechanism to hold any key value in memory // wrapped around via sync.Map but typed with generics. type Cache[K comparable, V any] struct { m sync.Map } // Delete delete the key func (r *Cache[K, V]) Delete(key K) { r.m.Delete(key) } // Get - Returns a value of a given key if it exists. func (r *Cache[K, V]) Get(key K) (value V, ok bool) { return r.load(key) } // Set - Will persist a value into cache. func (r *Cache[K, V]) Set(key K, value V) { r.store(key, value) } func (r *Cache[K, V]) load(key K) (V, bool) { value, ok := r.m.Load(key) if !ok { var zero V return zero, false } return value.(V), true } func (r *Cache[K, V]) store(key K, value V) { r.m.Store(key, value) } minio-go-7.0.97/pkg/lifecycle/000077500000000000000000000000001510244170000160465ustar00rootroot00000000000000minio-go-7.0.97/pkg/lifecycle/lifecycle.go000066400000000000000000000467411510244170000203500ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // Package lifecycle contains all the lifecycle related data types and marshallers. package lifecycle import ( "encoding/json" "encoding/xml" "errors" "time" ) var errMissingStorageClass = errors.New("storage-class cannot be empty") // AbortIncompleteMultipartUpload structure, not supported yet on MinIO type AbortIncompleteMultipartUpload struct { XMLName xml.Name `xml:"AbortIncompleteMultipartUpload,omitempty" json:"-"` DaysAfterInitiation ExpirationDays `xml:"DaysAfterInitiation,omitempty" json:"DaysAfterInitiation,omitempty"` } // IsDaysNull returns true if days field is null func (n AbortIncompleteMultipartUpload) IsDaysNull() bool { return n.DaysAfterInitiation == ExpirationDays(0) } // MarshalXML if days after initiation is set to non-zero value func (n AbortIncompleteMultipartUpload) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if n.IsDaysNull() { return nil } type abortIncompleteMultipartUploadWrapper AbortIncompleteMultipartUpload return e.EncodeElement(abortIncompleteMultipartUploadWrapper(n), start) } // NoncurrentVersionExpiration - Specifies when noncurrent object versions expire. // Upon expiration, server permanently deletes the noncurrent object versions. // Set this lifecycle configuration action on a bucket that has versioning enabled // (or suspended) to request server delete noncurrent object versions at a // specific period in the object's lifetime. type NoncurrentVersionExpiration struct { XMLName xml.Name `xml:"NoncurrentVersionExpiration" json:"-"` NoncurrentDays ExpirationDays `xml:"NoncurrentDays,omitempty" json:"NoncurrentDays,omitempty"` NewerNoncurrentVersions int `xml:"NewerNoncurrentVersions,omitempty" json:"NewerNoncurrentVersions,omitempty"` } // MarshalXML if n is non-empty, i.e has a non-zero NoncurrentDays or NewerNoncurrentVersions. func (n NoncurrentVersionExpiration) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if n.isNull() { return nil } type noncurrentVersionExpirationWrapper NoncurrentVersionExpiration return e.EncodeElement(noncurrentVersionExpirationWrapper(n), start) } // IsDaysNull returns true if days field is null func (n NoncurrentVersionExpiration) IsDaysNull() bool { return n.NoncurrentDays == ExpirationDays(0) } func (n NoncurrentVersionExpiration) isNull() bool { return n.IsDaysNull() && n.NewerNoncurrentVersions == 0 } // NoncurrentVersionTransition structure, set this action to request server to // transition noncurrent object versions to different set storage classes // at a specific period in the object's lifetime. type NoncurrentVersionTransition struct { XMLName xml.Name `xml:"NoncurrentVersionTransition,omitempty" json:"-"` StorageClass string `xml:"StorageClass,omitempty" json:"StorageClass,omitempty"` NoncurrentDays ExpirationDays `xml:"NoncurrentDays" json:"NoncurrentDays"` NewerNoncurrentVersions int `xml:"NewerNoncurrentVersions,omitempty" json:"NewerNoncurrentVersions,omitempty"` } // IsDaysNull returns true if days field is null func (n NoncurrentVersionTransition) IsDaysNull() bool { return n.NoncurrentDays == ExpirationDays(0) } // IsStorageClassEmpty returns true if storage class field is empty func (n NoncurrentVersionTransition) IsStorageClassEmpty() bool { return n.StorageClass == "" } func (n NoncurrentVersionTransition) isNull() bool { return n.StorageClass == "" } // UnmarshalJSON implements NoncurrentVersionTransition JSONify func (n *NoncurrentVersionTransition) UnmarshalJSON(b []byte) error { type noncurrentVersionTransition NoncurrentVersionTransition var nt noncurrentVersionTransition err := json.Unmarshal(b, &nt) if err != nil { return err } if nt.StorageClass == "" { return errMissingStorageClass } *n = NoncurrentVersionTransition(nt) return nil } // MarshalXML is extended to leave out // tags func (n NoncurrentVersionTransition) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if n.isNull() { return nil } type noncurrentVersionTransitionWrapper NoncurrentVersionTransition return e.EncodeElement(noncurrentVersionTransitionWrapper(n), start) } // Tag structure key/value pair representing an object tag to apply lifecycle configuration type Tag struct { XMLName xml.Name `xml:"Tag,omitempty" json:"-"` Key string `xml:"Key,omitempty" json:"Key,omitempty"` Value string `xml:"Value,omitempty" json:"Value,omitempty"` } // IsEmpty returns whether this tag is empty or not. func (tag Tag) IsEmpty() bool { return tag.Key == "" } // Transition structure - transition details of lifecycle configuration type Transition struct { XMLName xml.Name `xml:"Transition" json:"-"` Date ExpirationDate `xml:"Date,omitempty" json:"Date,omitempty"` StorageClass string `xml:"StorageClass,omitempty" json:"StorageClass,omitempty"` Days ExpirationDays `xml:"Days" json:"Days"` } // UnmarshalJSON returns an error if storage-class is empty. func (t *Transition) UnmarshalJSON(b []byte) error { type transition Transition var tr transition err := json.Unmarshal(b, &tr) if err != nil { return err } if tr.StorageClass == "" { return errMissingStorageClass } *t = Transition(tr) return nil } // MarshalJSON customizes json encoding by omitting empty values func (t Transition) MarshalJSON() ([]byte, error) { if t.IsNull() { return nil, nil } type transition struct { Date *ExpirationDate `json:"Date,omitempty"` StorageClass string `json:"StorageClass,omitempty"` Days *ExpirationDays `json:"Days"` } newt := transition{ StorageClass: t.StorageClass, } if !t.IsDateNull() { newt.Date = &t.Date } else { newt.Days = &t.Days } return json.Marshal(newt) } // IsDaysNull returns true if days field is null func (t Transition) IsDaysNull() bool { return t.Days == ExpirationDays(0) } // IsDateNull returns true if date field is null func (t Transition) IsDateNull() bool { return t.Date.IsZero() } // IsNull returns true if no storage-class is set. func (t Transition) IsNull() bool { return t.StorageClass == "" } // MarshalXML is transition is non null func (t Transition) MarshalXML(en *xml.Encoder, startElement xml.StartElement) error { if t.IsNull() { return nil } type transitionWrapper Transition return en.EncodeElement(transitionWrapper(t), startElement) } // And And Rule for LifecycleTag, to be used in LifecycleRuleFilter type And struct { XMLName xml.Name `xml:"And" json:"-"` Prefix string `xml:"Prefix" json:"Prefix,omitempty"` Tags []Tag `xml:"Tag" json:"Tags,omitempty"` ObjectSizeLessThan int64 `xml:"ObjectSizeLessThan,omitempty" json:"ObjectSizeLessThan,omitempty"` ObjectSizeGreaterThan int64 `xml:"ObjectSizeGreaterThan,omitempty" json:"ObjectSizeGreaterThan,omitempty"` } // IsEmpty returns true if Tags field is null func (a And) IsEmpty() bool { return len(a.Tags) == 0 && a.Prefix == "" && a.ObjectSizeLessThan == 0 && a.ObjectSizeGreaterThan == 0 } // Filter will be used in selecting rule(s) for lifecycle configuration type Filter struct { XMLName xml.Name `xml:"Filter" json:"-"` And And `xml:"And,omitempty" json:"And,omitempty"` Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` Tag Tag `xml:"Tag,omitempty" json:"Tag,omitempty"` ObjectSizeLessThan int64 `xml:"ObjectSizeLessThan,omitempty" json:"ObjectSizeLessThan,omitempty"` ObjectSizeGreaterThan int64 `xml:"ObjectSizeGreaterThan,omitempty" json:"ObjectSizeGreaterThan,omitempty"` } // IsNull returns true if all Filter fields are empty. func (f Filter) IsNull() bool { return f.Tag.IsEmpty() && f.And.IsEmpty() && f.Prefix == "" && f.ObjectSizeLessThan == 0 && f.ObjectSizeGreaterThan == 0 } // MarshalJSON customizes json encoding by removing empty values. func (f Filter) MarshalJSON() ([]byte, error) { type filter struct { And *And `json:"And,omitempty"` Prefix string `json:"Prefix,omitempty"` Tag *Tag `json:"Tag,omitempty"` ObjectSizeLessThan int64 `json:"ObjectSizeLessThan,omitempty"` ObjectSizeGreaterThan int64 `json:"ObjectSizeGreaterThan,omitempty"` } newf := filter{ Prefix: f.Prefix, } if !f.Tag.IsEmpty() { newf.Tag = &f.Tag } if !f.And.IsEmpty() { newf.And = &f.And } newf.ObjectSizeLessThan = f.ObjectSizeLessThan newf.ObjectSizeGreaterThan = f.ObjectSizeGreaterThan return json.Marshal(newf) } // MarshalXML - produces the xml representation of the Filter struct // only one of Prefix, And and Tag should be present in the output. func (f Filter) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if err := e.EncodeToken(start); err != nil { return err } switch { case !f.And.IsEmpty(): if err := e.EncodeElement(f.And, xml.StartElement{Name: xml.Name{Local: "And"}}); err != nil { return err } case !f.Tag.IsEmpty(): if err := e.EncodeElement(f.Tag, xml.StartElement{Name: xml.Name{Local: "Tag"}}); err != nil { return err } default: if f.ObjectSizeLessThan > 0 { if err := e.EncodeElement(f.ObjectSizeLessThan, xml.StartElement{Name: xml.Name{Local: "ObjectSizeLessThan"}}); err != nil { return err } break } if f.ObjectSizeGreaterThan > 0 { if err := e.EncodeElement(f.ObjectSizeGreaterThan, xml.StartElement{Name: xml.Name{Local: "ObjectSizeGreaterThan"}}); err != nil { return err } break } // Print empty Prefix field only when everything else is empty if err := e.EncodeElement(f.Prefix, xml.StartElement{Name: xml.Name{Local: "Prefix"}}); err != nil { return err } } return e.EncodeToken(xml.EndElement{Name: start.Name}) } // ExpirationDays is a type alias to unmarshal Days in Expiration type ExpirationDays int // MarshalXML encodes number of days to expire if it is non-zero and // encodes empty string otherwise func (eDays ExpirationDays) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { if eDays == 0 { return nil } return e.EncodeElement(int(eDays), startElement) } // ExpirationDate is a embedded type containing time.Time to unmarshal // Date in Expiration type ExpirationDate struct { time.Time } // MarshalXML encodes expiration date if it is non-zero and encodes // empty string otherwise func (eDate ExpirationDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { if eDate.IsZero() { return nil } return e.EncodeElement(eDate.Format(time.RFC3339), startElement) } // ExpireDeleteMarker represents value of ExpiredObjectDeleteMarker field in Expiration XML element. type ExpireDeleteMarker ExpirationBoolean // IsEnabled returns true if the auto delete-marker expiration is enabled func (e ExpireDeleteMarker) IsEnabled() bool { return bool(e) } // ExpirationBoolean represents an XML version of 'bool' type type ExpirationBoolean bool // MarshalXML encodes delete marker boolean into an XML form. func (b ExpirationBoolean) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { if !b { return nil } type booleanWrapper ExpirationBoolean return e.EncodeElement(booleanWrapper(b), startElement) } // IsEnabled returns true if the expiration boolean is enabled func (b ExpirationBoolean) IsEnabled() bool { return bool(b) } // Expiration structure - expiration details of lifecycle configuration type Expiration struct { XMLName xml.Name `xml:"Expiration,omitempty" json:"-"` Date ExpirationDate `xml:"Date,omitempty" json:"Date,omitempty"` Days ExpirationDays `xml:"Days,omitempty" json:"Days,omitempty"` DeleteMarker ExpireDeleteMarker `xml:"ExpiredObjectDeleteMarker,omitempty" json:"ExpiredObjectDeleteMarker,omitempty"` DeleteAll ExpirationBoolean `xml:"ExpiredObjectAllVersions,omitempty" json:"ExpiredObjectAllVersions,omitempty"` } // MarshalJSON customizes json encoding by removing empty day/date specification. func (e Expiration) MarshalJSON() ([]byte, error) { type expiration struct { Date *ExpirationDate `json:"Date,omitempty"` Days *ExpirationDays `json:"Days,omitempty"` DeleteMarker ExpireDeleteMarker `json:"ExpiredObjectDeleteMarker,omitempty"` DeleteAll ExpirationBoolean `json:"ExpiredObjectAllVersions,omitempty"` } newexp := expiration{ DeleteMarker: e.DeleteMarker, DeleteAll: e.DeleteAll, } if !e.IsDaysNull() { newexp.Days = &e.Days } if !e.IsDateNull() { newexp.Date = &e.Date } return json.Marshal(newexp) } // IsDaysNull returns true if days field is null func (e Expiration) IsDaysNull() bool { return e.Days == ExpirationDays(0) } // IsDateNull returns true if date field is null func (e Expiration) IsDateNull() bool { return e.Date.IsZero() } // IsDeleteMarkerExpirationEnabled returns true if the auto-expiration of delete marker is enabled func (e Expiration) IsDeleteMarkerExpirationEnabled() bool { return e.DeleteMarker.IsEnabled() } // IsNull returns true if both date and days fields are null func (e Expiration) IsNull() bool { return e.IsDaysNull() && e.IsDateNull() && !e.IsDeleteMarkerExpirationEnabled() && !e.DeleteAll.IsEnabled() } // MarshalXML is expiration is non null func (e Expiration) MarshalXML(en *xml.Encoder, startElement xml.StartElement) error { if e.IsNull() { return nil } type expirationWrapper Expiration return en.EncodeElement(expirationWrapper(e), startElement) } // DelMarkerExpiration represents DelMarkerExpiration actions element in an ILM policy type DelMarkerExpiration struct { XMLName xml.Name `xml:"DelMarkerExpiration" json:"-"` Days int `xml:"Days,omitempty" json:"Days,omitempty"` } // IsNull returns true if Days isn't specified and false otherwise. func (de DelMarkerExpiration) IsNull() bool { return de.Days == 0 } // MarshalXML avoids serializing an empty DelMarkerExpiration element func (de DelMarkerExpiration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { if de.IsNull() { return nil } type delMarkerExp DelMarkerExpiration return enc.EncodeElement(delMarkerExp(de), start) } // AllVersionsExpiration represents AllVersionsExpiration actions element in an ILM policy type AllVersionsExpiration struct { XMLName xml.Name `xml:"AllVersionsExpiration" json:"-"` Days int `xml:"Days,omitempty" json:"Days,omitempty"` DeleteMarker ExpireDeleteMarker `xml:"DeleteMarker,omitempty" json:"DeleteMarker,omitempty"` } // IsNull returns true if days field is 0 func (e AllVersionsExpiration) IsNull() bool { return e.Days == 0 } // MarshalXML satisfies xml.Marshaler to provide custom encoding func (e AllVersionsExpiration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { if e.IsNull() { return nil } type allVersionsExp AllVersionsExpiration return enc.EncodeElement(allVersionsExp(e), start) } // MarshalJSON customizes json encoding by omitting empty values func (r Rule) MarshalJSON() ([]byte, error) { type rule struct { AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `json:"AbortIncompleteMultipartUpload,omitempty"` Expiration *Expiration `json:"Expiration,omitempty"` DelMarkerExpiration *DelMarkerExpiration `json:"DelMarkerExpiration,omitempty"` AllVersionsExpiration *AllVersionsExpiration `json:"AllVersionsExpiration,omitempty"` ID string `json:"ID"` RuleFilter *Filter `json:"Filter,omitempty"` NoncurrentVersionExpiration *NoncurrentVersionExpiration `json:"NoncurrentVersionExpiration,omitempty"` NoncurrentVersionTransition *NoncurrentVersionTransition `json:"NoncurrentVersionTransition,omitempty"` Prefix string `json:"Prefix,omitempty"` Status string `json:"Status"` Transition *Transition `json:"Transition,omitempty"` } newr := rule{ Prefix: r.Prefix, Status: r.Status, ID: r.ID, } if !r.RuleFilter.IsNull() { newr.RuleFilter = &r.RuleFilter } if !r.AbortIncompleteMultipartUpload.IsDaysNull() { newr.AbortIncompleteMultipartUpload = &r.AbortIncompleteMultipartUpload } if !r.Expiration.IsNull() { newr.Expiration = &r.Expiration } if !r.DelMarkerExpiration.IsNull() { newr.DelMarkerExpiration = &r.DelMarkerExpiration } if !r.Transition.IsNull() { newr.Transition = &r.Transition } if !r.NoncurrentVersionExpiration.isNull() { newr.NoncurrentVersionExpiration = &r.NoncurrentVersionExpiration } if !r.NoncurrentVersionTransition.isNull() { newr.NoncurrentVersionTransition = &r.NoncurrentVersionTransition } if !r.AllVersionsExpiration.IsNull() { newr.AllVersionsExpiration = &r.AllVersionsExpiration } return json.Marshal(newr) } // Rule represents a single rule in lifecycle configuration type Rule struct { XMLName xml.Name `xml:"Rule,omitempty" json:"-"` AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty" json:"AbortIncompleteMultipartUpload,omitempty"` Expiration Expiration `xml:"Expiration,omitempty" json:"Expiration,omitempty"` DelMarkerExpiration DelMarkerExpiration `xml:"DelMarkerExpiration,omitempty" json:"DelMarkerExpiration,omitempty"` AllVersionsExpiration AllVersionsExpiration `xml:"AllVersionsExpiration,omitempty" json:"AllVersionsExpiration,omitempty"` ID string `xml:"ID" json:"ID"` RuleFilter Filter `xml:"Filter,omitempty" json:"Filter,omitempty"` NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty" json:"NoncurrentVersionExpiration,omitempty"` NoncurrentVersionTransition NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty" json:"NoncurrentVersionTransition,omitempty"` Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` Status string `xml:"Status" json:"Status"` Transition Transition `xml:"Transition,omitempty" json:"Transition,omitempty"` } // Configuration is a collection of Rule objects. type Configuration struct { XMLName xml.Name `xml:"LifecycleConfiguration,omitempty" json:"-"` Rules []Rule `xml:"Rule"` } // Empty check if lifecycle configuration is empty func (c *Configuration) Empty() bool { if c == nil { return true } return len(c.Rules) == 0 } // NewConfiguration initializes a fresh lifecycle configuration // for manipulation, such as setting and removing lifecycle rules // and filters. func NewConfiguration() *Configuration { return &Configuration{} } minio-go-7.0.97/pkg/lifecycle/lifecycle_test.go000066400000000000000000000275351510244170000214070ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2021 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package lifecycle import ( "bytes" "encoding/json" "encoding/xml" "testing" "time" ) func TestLifecycleUnmarshalJSON(t *testing.T) { testCases := []struct { input string err error }{ { input: `{ "Rules": [ { "ID": "transition-missing", "Status": "Enabled", "Transition": { "Days": 0 } } ] }`, err: errMissingStorageClass, }, { input: `{ "Rules": [ { "ID": "transition-missing-1", "Status": "Enabled", "Transition": { "Days": 1 } } ] }`, err: errMissingStorageClass, }, { input: `{ "Rules": [ { "ID": "noncurrent-transition-missing", "Status": "Enabled", "NoncurrentVersionTransition": { "NoncurrentDays": 0 } } ] }`, err: errMissingStorageClass, }, { input: `{ "Rules": [ { "ID": "noncurrent-transition-missing-1", "Status": "Enabled", "NoncurrentVersionTransition": { "NoncurrentDays": 1 } } ] }`, err: errMissingStorageClass, }, { input: `{ "Rules": [ { "ID": "transition", "Status": "Enabled", "Transition": { "StorageClass": "S3TIER-1", "Days": 1 } } ] }`, err: nil, }, { input: `{ "Rules": [ { "ID": "noncurrent-transition", "Status": "Enabled", "NoncurrentVersionTransition": { "StorageClass": "S3TIER-1", "NoncurrentDays": 1 } } ] }`, err: nil, }, { input: `{ "Rules": [ { "ID": "transition-lt", "Status": "Enabled", "Filter": { "ObjectSizeLessThan": 1048576 }, "Transition": { "StorageClass": "S3TIER-1", "Days": 1 } } ] }`, err: nil, }, { input: `{ "Rules": [ { "ID": "noncurrent-transition-gt", "Status": "Enabled", "Filter": { "ObjectSizeGreaterThan": 10485760 }, "NoncurrentVersionTransition": { "StorageClass": "S3TIER-1", "NoncurrentDays": 1 } } ] }`, err: nil, }, { input: `{ "Rules": [ { "ID": "noncurrent-transition-lt-and-gt", "Status": "Enabled", "Filter": { "And": { "ObjectSizeGreaterThan": 10485760, "ObjectSizeLessThan": 1048576 } }, "NoncurrentVersionTransition": { "StorageClass": "S3TIER-1", "NoncurrentDays": 1 } } ] }`, err: nil, }, } for i, tc := range testCases { var lc Configuration if err := json.Unmarshal([]byte(tc.input), &lc); err != tc.err { t.Fatalf("%d: expected error %v but got %v", i+1, tc.err, err) } } } func TestLifecycleJSONRoundtrip(t *testing.T) { testNow := time.Date(2021, time.January, 1, 0, 0, 0, 0, time.UTC) lc := Configuration{ Rules: []Rule{ { RuleFilter: Filter{ Prefix: "prefix", }, Expiration: Expiration{ Days: ExpirationDays(3), }, AbortIncompleteMultipartUpload: AbortIncompleteMultipartUpload{ DaysAfterInitiation: ExpirationDays(1), }, ID: "rule-1", Status: "Enabled", }, { RuleFilter: Filter{ And: And{ Prefix: "prefix", Tags: []Tag{ { Key: "key-1", Value: "val-1", }, }, }, }, Expiration: Expiration{ Date: ExpirationDate{ testNow, }, }, NoncurrentVersionExpiration: NoncurrentVersionExpiration{ NoncurrentDays: ExpirationDays(1), }, ID: "rule-2", Status: "Enabled", }, { Transition: Transition{ Days: ExpirationDays(3), StorageClass: "MINIOTIER-1", }, Expiration: Expiration{ DeleteMarker: ExpireDeleteMarker(true), }, NoncurrentVersionTransition: NoncurrentVersionTransition{ NoncurrentDays: ExpirationDays(3), StorageClass: "MINIOTIER-2", }, ID: "rule-3", Status: "Enabled", }, { Transition: Transition{ Date: ExpirationDate{testNow}, StorageClass: "MINIOTIER-1", }, ID: "rule-4", Status: "Enabled", }, { NoncurrentVersionExpiration: NoncurrentVersionExpiration{ NoncurrentDays: ExpirationDays(3), NewerNoncurrentVersions: 1, }, NoncurrentVersionTransition: NoncurrentVersionTransition{ NoncurrentDays: ExpirationDays(3), NewerNoncurrentVersions: 1, StorageClass: "MINIOTIER-2", }, ID: "rule-5", Status: "Enabled", }, { Expiration: Expiration{ DeleteMarker: true, }, ID: "rule-6", Status: "Enabled", }, { DelMarkerExpiration: DelMarkerExpiration{ Days: 10, }, ID: "rule-7", Status: "Enabled", }, { AllVersionsExpiration: AllVersionsExpiration{ Days: 10, }, ID: "rule-8", Status: "Enabled", }, { AllVersionsExpiration: AllVersionsExpiration{ Days: 0, }, ID: "rule-9", Status: "Enabled", }, { AllVersionsExpiration: AllVersionsExpiration{ Days: 7, DeleteMarker: ExpireDeleteMarker(true), }, ID: "rule-10", Status: "Enabled", }, }, } buf, err := json.Marshal(lc) if err != nil { t.Fatal("failed to marshal json", err) } var got Configuration if err = json.Unmarshal(buf, &got); err != nil { t.Fatal("failed to unmarshal json", err) } for i := range lc.Rules { if !lc.Rules[i].NoncurrentVersionTransition.equals(got.Rules[i].NoncurrentVersionTransition) { t.Fatalf("expected %#v got %#v", lc.Rules[i].NoncurrentVersionTransition, got.Rules[i].NoncurrentVersionTransition) } if !lc.Rules[i].NoncurrentVersionExpiration.equals(got.Rules[i].NoncurrentVersionExpiration) { t.Fatalf("expected %#v got %#v", lc.Rules[i].NoncurrentVersionExpiration, got.Rules[i].NoncurrentVersionExpiration) } if !lc.Rules[i].Transition.equals(got.Rules[i].Transition) { t.Fatalf("expected %#v got %#v", lc.Rules[i].Transition, got.Rules[i].Transition) } if lc.Rules[i].Expiration != got.Rules[i].Expiration { t.Fatalf("expected %#v got %#v", lc.Rules[i].Expiration, got.Rules[i].Expiration) } if !lc.Rules[i].DelMarkerExpiration.equals(got.Rules[i].DelMarkerExpiration) { t.Fatalf("expected %#v got %#v", lc.Rules[i].DelMarkerExpiration, got.Rules[i].DelMarkerExpiration) } if !lc.Rules[i].AllVersionsExpiration.equals(got.Rules[i].AllVersionsExpiration) { t.Fatalf("expected %#v got %#v", lc.Rules[i].AllVersionsExpiration, got.Rules[i].AllVersionsExpiration) } } } func TestLifecycleXMLRoundtrip(t *testing.T) { lc := Configuration{ Rules: []Rule{ { ID: "immediate-noncurrent", Status: "Enabled", NoncurrentVersionTransition: NoncurrentVersionTransition{ NoncurrentDays: 0, StorageClass: "S3TIER-1", }, }, { ID: "immediate-current", Status: "Enabled", Transition: Transition{ StorageClass: "S3TIER-1", Days: 0, }, }, { ID: "current", Status: "Enabled", Transition: Transition{ StorageClass: "S3TIER-1", Date: ExpirationDate{time.Date(2021, time.September, 1, 0, 0, 0, 0, time.UTC)}, }, }, { ID: "noncurrent", Status: "Enabled", NoncurrentVersionTransition: NoncurrentVersionTransition{ NoncurrentDays: ExpirationDays(5), StorageClass: "S3TIER-1", }, }, { ID: "max-noncurrent-versions", Status: "Enabled", NoncurrentVersionExpiration: NoncurrentVersionExpiration{ NewerNoncurrentVersions: 5, }, }, { ID: "delmarker-expiration", Status: "Enabled", DelMarkerExpiration: DelMarkerExpiration{ Days: 5, }, }, { ID: "all-versions-expiration-1", Status: "Enabled", AllVersionsExpiration: AllVersionsExpiration{ Days: 5, }, }, { ID: "all-versions-expiration-2", Status: "Enabled", AllVersionsExpiration: AllVersionsExpiration{ Days: 10, DeleteMarker: ExpireDeleteMarker(true), }, RuleFilter: Filter{ Tag: Tag{ Key: "key-1", Value: "value-1", }, }, }, }, } buf, err := xml.Marshal(lc) if err != nil { t.Fatalf("failed to marshal lifecycle configuration %v", err) } var got Configuration err = xml.Unmarshal(buf, &got) if err != nil { t.Fatalf("failed to unmarshal lifecycle %v", err) } for i := range lc.Rules { if !lc.Rules[i].NoncurrentVersionTransition.equals(got.Rules[i].NoncurrentVersionTransition) { t.Fatalf("%d: expected %#v got %#v", i+1, lc.Rules[i].NoncurrentVersionTransition, got.Rules[i].NoncurrentVersionTransition) } if !lc.Rules[i].Transition.equals(got.Rules[i].Transition) { t.Fatalf("%d: expected %#v got %#v", i+1, lc.Rules[i].Transition, got.Rules[i].Transition) } if !lc.Rules[i].NoncurrentVersionExpiration.equals(got.Rules[i].NoncurrentVersionExpiration) { t.Fatalf("%d: expected %#v got %#v", i+1, lc.Rules[i].NoncurrentVersionExpiration, got.Rules[i].NoncurrentVersionExpiration) } if !lc.Rules[i].DelMarkerExpiration.equals(got.Rules[i].DelMarkerExpiration) { t.Fatalf("%d: expected %#v got %#v", i+1, lc.Rules[i].DelMarkerExpiration, got.Rules[i].DelMarkerExpiration) } if !lc.Rules[i].AllVersionsExpiration.equals(got.Rules[i].AllVersionsExpiration) { t.Fatalf("%d: expected %#v got %#v", i+1, lc.Rules[i].AllVersionsExpiration, got.Rules[i].AllVersionsExpiration) } } } func (n NoncurrentVersionTransition) equals(m NoncurrentVersionTransition) bool { return n.NoncurrentDays == m.NoncurrentDays && n.StorageClass == m.StorageClass } func (n NoncurrentVersionExpiration) equals(m NoncurrentVersionExpiration) bool { return n.NoncurrentDays == m.NoncurrentDays && n.NewerNoncurrentVersions == m.NewerNoncurrentVersions } func (t Transition) equals(u Transition) bool { return t.Days == u.Days && t.Date.Equal(u.Date.Time) && t.StorageClass == u.StorageClass } func (a DelMarkerExpiration) equals(b DelMarkerExpiration) bool { return a.Days == b.Days } func (a AllVersionsExpiration) equals(b AllVersionsExpiration) bool { return a.Days == b.Days && a.DeleteMarker == b.DeleteMarker } func TestExpiredObjectDeleteMarker(t *testing.T) { expected := []byte(`{"Rules":[{"Expiration":{"ExpiredObjectDeleteMarker":true},"ID":"expired-object-delete-marker","Status":"Enabled"}]}`) lc := Configuration{ Rules: []Rule{ { Expiration: Expiration{ DeleteMarker: true, }, ID: "expired-object-delete-marker", Status: "Enabled", }, }, } got, err := json.Marshal(lc) if err != nil { t.Fatalf("Failed to marshal due to %v", err) } if !bytes.Equal(expected, got) { t.Fatalf("Expected %s but got %s", expected, got) } } func TestAllVersionsExpiration(t *testing.T) { expected := []byte(`{"Rules":[{"AllVersionsExpiration":{"Days":2,"DeleteMarker":true},"ID":"all-versions-expiration","Status":"Enabled"}]}`) lc := Configuration{ Rules: []Rule{ { AllVersionsExpiration: AllVersionsExpiration{ Days: 2, DeleteMarker: ExpireDeleteMarker(true), }, ID: "all-versions-expiration", Status: "Enabled", }, }, } got, err := json.Marshal(lc) if err != nil { t.Fatalf("Failed to marshal due to %v", err) } if !bytes.Equal(expected, got) { t.Fatalf("Expected %s but got %s", expected, got) } } minio-go-7.0.97/pkg/notification/000077500000000000000000000000001510244170000165755ustar00rootroot00000000000000minio-go-7.0.97/pkg/notification/info.go000066400000000000000000000053261510244170000200650ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package notification // Indentity represents the user id, this is a compliance field. type identity struct { PrincipalID string `json:"principalId"` } // event bucket metadata. type bucketMeta struct { Name string `json:"name"` OwnerIdentity identity `json:"ownerIdentity"` ARN string `json:"arn"` } // event object metadata. type objectMeta struct { Key string `json:"key"` Size int64 `json:"size,omitempty"` ETag string `json:"eTag,omitempty"` ContentType string `json:"contentType,omitempty"` UserMetadata map[string]string `json:"userMetadata,omitempty"` VersionID string `json:"versionId,omitempty"` Sequencer string `json:"sequencer"` } // event server specific metadata. type eventMeta struct { SchemaVersion string `json:"s3SchemaVersion"` ConfigurationID string `json:"configurationId"` Bucket bucketMeta `json:"bucket"` Object objectMeta `json:"object"` } // sourceInfo represents information on the client that // triggered the event notification. type sourceInfo struct { Host string `json:"host"` Port string `json:"port"` UserAgent string `json:"userAgent"` } // Event represents an Amazon an S3 bucket notification event. type Event struct { EventVersion string `json:"eventVersion"` EventSource string `json:"eventSource"` AwsRegion string `json:"awsRegion"` EventTime string `json:"eventTime"` EventName string `json:"eventName"` UserIdentity identity `json:"userIdentity"` RequestParameters map[string]string `json:"requestParameters"` ResponseElements map[string]string `json:"responseElements"` S3 eventMeta `json:"s3"` Source sourceInfo `json:"source"` } // Info - represents the collection of notification events, additionally // also reports errors if any while listening on bucket notifications. type Info struct { Records []Event Err error } minio-go-7.0.97/pkg/notification/notification.go000066400000000000000000000370211510244170000216150ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package notification import ( "encoding/xml" "errors" "fmt" "strings" "github.com/minio/minio-go/v7/pkg/set" ) // EventType is a S3 notification event associated to the bucket notification configuration type EventType string // The role of all event types are described in : // // http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html#notification-how-to-event-types-and-destinations const ( ObjectCreatedAll EventType = "s3:ObjectCreated:*" ObjectCreatedPut EventType = "s3:ObjectCreated:Put" ObjectCreatedPost EventType = "s3:ObjectCreated:Post" ObjectCreatedCopy EventType = "s3:ObjectCreated:Copy" ObjectCreatedDeleteTagging EventType = "s3:ObjectCreated:DeleteTagging" ObjectCreatedCompleteMultipartUpload EventType = "s3:ObjectCreated:CompleteMultipartUpload" ObjectCreatedPutLegalHold EventType = "s3:ObjectCreated:PutLegalHold" ObjectCreatedPutRetention EventType = "s3:ObjectCreated:PutRetention" ObjectCreatedPutTagging EventType = "s3:ObjectCreated:PutTagging" ObjectAccessedGet EventType = "s3:ObjectAccessed:Get" ObjectAccessedHead EventType = "s3:ObjectAccessed:Head" ObjectAccessedGetRetention EventType = "s3:ObjectAccessed:GetRetention" ObjectAccessedGetLegalHold EventType = "s3:ObjectAccessed:GetLegalHold" ObjectAccessedAll EventType = "s3:ObjectAccessed:*" ObjectRemovedAll EventType = "s3:ObjectRemoved:*" ObjectRemovedDelete EventType = "s3:ObjectRemoved:Delete" ObjectRemovedDeleteMarkerCreated EventType = "s3:ObjectRemoved:DeleteMarkerCreated" ILMDelMarkerExpirationDelete EventType = "s3:LifecycleDelMarkerExpiration:Delete" ObjectReducedRedundancyLostObject EventType = "s3:ReducedRedundancyLostObject" ObjectTransitionAll EventType = "s3:ObjectTransition:*" ObjectTransitionFailed EventType = "s3:ObjectTransition:Failed" ObjectTransitionComplete EventType = "s3:ObjectTransition:Complete" ObjectTransitionPost EventType = "s3:ObjectRestore:Post" ObjectTransitionCompleted EventType = "s3:ObjectRestore:Completed" ObjectReplicationAll EventType = "s3:Replication:*" ObjectReplicationOperationCompletedReplication EventType = "s3:Replication:OperationCompletedReplication" ObjectReplicationOperationFailedReplication EventType = "s3:Replication:OperationFailedReplication" ObjectReplicationOperationMissedThreshold EventType = "s3:Replication:OperationMissedThreshold" ObjectReplicationOperationNotTracked EventType = "s3:Replication:OperationNotTracked" ObjectReplicationOperationReplicatedAfterThreshold EventType = "s3:Replication:OperationReplicatedAfterThreshold" ObjectScannerManyVersions EventType = "s3:Scanner:ManyVersions" ObjectScannerBigPrefix EventType = "s3:Scanner:BigPrefix" ObjectScannerAll EventType = "s3:Scanner:*" BucketCreatedAll EventType = "s3:BucketCreated:*" BucketRemovedAll EventType = "s3:BucketRemoved:*" ) // FilterRule - child of S3Key, a tag in the notification xml which // carries suffix/prefix filters type FilterRule struct { Name string `xml:"Name"` Value string `xml:"Value"` } // S3Key - child of Filter, a tag in the notification xml which // carries suffix/prefix filters type S3Key struct { FilterRules []FilterRule `xml:"FilterRule,omitempty"` } // Filter - a tag in the notification xml structure which carries // suffix/prefix filters type Filter struct { S3Key S3Key `xml:"S3Key,omitempty"` } // Arn - holds ARN information that will be sent to the web service, // ARN desciption can be found in http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html type Arn struct { Partition string Service string Region string AccountID string Resource string } // NewArn creates new ARN based on the given partition, service, region, account id and resource func NewArn(partition, service, region, accountID, resource string) Arn { return Arn{ Partition: partition, Service: service, Region: region, AccountID: accountID, Resource: resource, } } var ( // ErrInvalidArnPrefix is returned when ARN string format does not start with 'arn' ErrInvalidArnPrefix = errors.New("invalid ARN format, must start with 'arn:'") // ErrInvalidArnFormat is returned when ARN string format is not valid ErrInvalidArnFormat = errors.New("invalid ARN format, must be 'arn:::::'") ) // NewArnFromString parses string representation of ARN into Arn object. // Returns an error if the string format is incorrect. func NewArnFromString(arn string) (Arn, error) { parts := strings.Split(arn, ":") if len(parts) != 6 { return Arn{}, ErrInvalidArnFormat } if parts[0] != "arn" { return Arn{}, ErrInvalidArnPrefix } return NewArn(parts[1], parts[2], parts[3], parts[4], parts[5]), nil } // String returns the string format of the ARN func (arn Arn) String() string { return "arn:" + arn.Partition + ":" + arn.Service + ":" + arn.Region + ":" + arn.AccountID + ":" + arn.Resource } // Config - represents one single notification configuration // such as topic, queue or lambda configuration. type Config struct { ID string `xml:"Id,omitempty"` Arn Arn `xml:"-"` Events []EventType `xml:"Event"` Filter *Filter `xml:"Filter,omitempty"` } // NewConfig creates one notification config and sets the given ARN func NewConfig(arn Arn) Config { return Config{Arn: arn, Filter: &Filter{}} } // AddEvents adds one event to the current notification config func (t *Config) AddEvents(events ...EventType) { t.Events = append(t.Events, events...) } // AddFilterSuffix sets the suffix configuration to the current notification config func (t *Config) AddFilterSuffix(suffix string) { if t.Filter == nil { t.Filter = &Filter{} } newFilterRule := FilterRule{Name: "suffix", Value: suffix} // Replace any suffix rule if existing and add to the list otherwise for index := range t.Filter.S3Key.FilterRules { if t.Filter.S3Key.FilterRules[index].Name == "suffix" { t.Filter.S3Key.FilterRules[index] = newFilterRule return } } t.Filter.S3Key.FilterRules = append(t.Filter.S3Key.FilterRules, newFilterRule) } // AddFilterPrefix sets the prefix configuration to the current notification config func (t *Config) AddFilterPrefix(prefix string) { if t.Filter == nil { t.Filter = &Filter{} } newFilterRule := FilterRule{Name: "prefix", Value: prefix} // Replace any prefix rule if existing and add to the list otherwise for index := range t.Filter.S3Key.FilterRules { if t.Filter.S3Key.FilterRules[index].Name == "prefix" { t.Filter.S3Key.FilterRules[index] = newFilterRule return } } t.Filter.S3Key.FilterRules = append(t.Filter.S3Key.FilterRules, newFilterRule) } // EqualEventTypeList tells whether a and b contain the same events func EqualEventTypeList(a, b []EventType) bool { if len(a) != len(b) { return false } setA := set.NewStringSet() for _, i := range a { setA.Add(string(i)) } setB := set.NewStringSet() for _, i := range b { setB.Add(string(i)) } return setA.Difference(setB).IsEmpty() } // EqualFilterRuleList tells whether a and b contain the same filters func EqualFilterRuleList(a, b []FilterRule) bool { if len(a) != len(b) { return false } setA := set.NewStringSet() for _, i := range a { setA.Add(fmt.Sprintf("%s-%s", i.Name, i.Value)) } setB := set.NewStringSet() for _, i := range b { setB.Add(fmt.Sprintf("%s-%s", i.Name, i.Value)) } return setA.Difference(setB).IsEmpty() } // Equal returns whether this `Config` is equal to another defined by the passed parameters func (t *Config) Equal(events []EventType, prefix, suffix string) bool { if t == nil { return false } // Compare events passEvents := EqualEventTypeList(t.Events, events) // Compare filters var newFilterRules []FilterRule if prefix != "" { newFilterRules = append(newFilterRules, FilterRule{Name: "prefix", Value: prefix}) } if suffix != "" { newFilterRules = append(newFilterRules, FilterRule{Name: "suffix", Value: suffix}) } var currentFilterRules []FilterRule if t.Filter != nil { currentFilterRules = t.Filter.S3Key.FilterRules } passFilters := EqualFilterRuleList(currentFilterRules, newFilterRules) return passEvents && passFilters } // TopicConfig carries one single topic notification configuration type TopicConfig struct { Config Topic string `xml:"Topic"` } // QueueConfig carries one single queue notification configuration type QueueConfig struct { Config Queue string `xml:"Queue"` } // LambdaConfig carries one single cloudfunction notification configuration type LambdaConfig struct { Config Lambda string `xml:"CloudFunction"` } // Configuration - the struct that represents the whole XML to be sent to the web service type Configuration struct { XMLName xml.Name `xml:"NotificationConfiguration"` LambdaConfigs []LambdaConfig `xml:"CloudFunctionConfiguration"` TopicConfigs []TopicConfig `xml:"TopicConfiguration"` QueueConfigs []QueueConfig `xml:"QueueConfiguration"` } // AddTopic adds a given topic config to the general bucket notification config func (b *Configuration) AddTopic(topicConfig Config) bool { newTopicConfig := TopicConfig{Config: topicConfig, Topic: topicConfig.Arn.String()} for _, n := range b.TopicConfigs { // If new config matches existing one if n.Topic == newTopicConfig.Arn.String() && newTopicConfig.Filter == n.Filter { existingConfig := set.NewStringSet() for _, v := range n.Events { existingConfig.Add(string(v)) } newConfig := set.NewStringSet() for _, v := range topicConfig.Events { newConfig.Add(string(v)) } if !newConfig.Intersection(existingConfig).IsEmpty() { return false } } } b.TopicConfigs = append(b.TopicConfigs, newTopicConfig) return true } // AddQueue adds a given queue config to the general bucket notification config func (b *Configuration) AddQueue(queueConfig Config) bool { newQueueConfig := QueueConfig{Config: queueConfig, Queue: queueConfig.Arn.String()} for _, n := range b.QueueConfigs { if n.Queue == newQueueConfig.Arn.String() && newQueueConfig.Filter == n.Filter { existingConfig := set.NewStringSet() for _, v := range n.Events { existingConfig.Add(string(v)) } newConfig := set.NewStringSet() for _, v := range queueConfig.Events { newConfig.Add(string(v)) } if !newConfig.Intersection(existingConfig).IsEmpty() { return false } } } b.QueueConfigs = append(b.QueueConfigs, newQueueConfig) return true } // AddLambda adds a given lambda config to the general bucket notification config func (b *Configuration) AddLambda(lambdaConfig Config) bool { newLambdaConfig := LambdaConfig{Config: lambdaConfig, Lambda: lambdaConfig.Arn.String()} for _, n := range b.LambdaConfigs { if n.Lambda == newLambdaConfig.Arn.String() && newLambdaConfig.Filter == n.Filter { existingConfig := set.NewStringSet() for _, v := range n.Events { existingConfig.Add(string(v)) } newConfig := set.NewStringSet() for _, v := range lambdaConfig.Events { newConfig.Add(string(v)) } if !newConfig.Intersection(existingConfig).IsEmpty() { return false } } } b.LambdaConfigs = append(b.LambdaConfigs, newLambdaConfig) return true } // RemoveTopicByArn removes all topic configurations that match the exact specified ARN func (b *Configuration) RemoveTopicByArn(arn Arn) { var topics []TopicConfig for _, topic := range b.TopicConfigs { if topic.Topic != arn.String() { topics = append(topics, topic) } } b.TopicConfigs = topics } // ErrNoConfigMatch is returned when a notification configuration (sqs,sns,lambda) is not found when trying to delete var ErrNoConfigMatch = errors.New("no notification configuration matched") // RemoveTopicByArnEventsPrefixSuffix removes a topic configuration that match the exact specified ARN, events, prefix and suffix func (b *Configuration) RemoveTopicByArnEventsPrefixSuffix(arn Arn, events []EventType, prefix, suffix string) error { removeIndex := -1 for i, v := range b.TopicConfigs { // if it matches events and filters, mark the index for deletion if v.Topic == arn.String() && v.Equal(events, prefix, suffix) { removeIndex = i break // since we have at most one matching config } } if removeIndex >= 0 { b.TopicConfigs = append(b.TopicConfigs[:removeIndex], b.TopicConfigs[removeIndex+1:]...) return nil } return ErrNoConfigMatch } // RemoveQueueByArn removes all queue configurations that match the exact specified ARN func (b *Configuration) RemoveQueueByArn(arn Arn) { var queues []QueueConfig for _, queue := range b.QueueConfigs { if queue.Queue != arn.String() { queues = append(queues, queue) } } b.QueueConfigs = queues } // RemoveQueueByArnEventsPrefixSuffix removes a queue configuration that match the exact specified ARN, events, prefix and suffix func (b *Configuration) RemoveQueueByArnEventsPrefixSuffix(arn Arn, events []EventType, prefix, suffix string) error { removeIndex := -1 for i, v := range b.QueueConfigs { // if it matches events and filters, mark the index for deletion if v.Queue == arn.String() && v.Equal(events, prefix, suffix) { removeIndex = i break // since we have at most one matching config } } if removeIndex >= 0 { b.QueueConfigs = append(b.QueueConfigs[:removeIndex], b.QueueConfigs[removeIndex+1:]...) return nil } return ErrNoConfigMatch } // RemoveLambdaByArn removes all lambda configurations that match the exact specified ARN func (b *Configuration) RemoveLambdaByArn(arn Arn) { var lambdas []LambdaConfig for _, lambda := range b.LambdaConfigs { if lambda.Lambda != arn.String() { lambdas = append(lambdas, lambda) } } b.LambdaConfigs = lambdas } // RemoveLambdaByArnEventsPrefixSuffix removes a topic configuration that match the exact specified ARN, events, prefix and suffix func (b *Configuration) RemoveLambdaByArnEventsPrefixSuffix(arn Arn, events []EventType, prefix, suffix string) error { removeIndex := -1 for i, v := range b.LambdaConfigs { // if it matches events and filters, mark the index for deletion if v.Lambda == arn.String() && v.Equal(events, prefix, suffix) { removeIndex = i break // since we have at most one matching config } } if removeIndex >= 0 { b.LambdaConfigs = append(b.LambdaConfigs[:removeIndex], b.LambdaConfigs[removeIndex+1:]...) return nil } return ErrNoConfigMatch } minio-go-7.0.97/pkg/notification/notification_test.go000066400000000000000000000726501510244170000226630ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package notification import ( "encoding/xml" "testing" ) func TestEqualEventTypeList(t *testing.T) { type args struct { a []EventType b []EventType } tests := []struct { name string args args want bool }{ { name: "same order", args: args{ a: []EventType{ObjectCreatedAll, ObjectAccessedAll}, b: []EventType{ObjectCreatedAll, ObjectAccessedAll}, }, want: true, }, { name: "different order", args: args{ a: []EventType{ObjectCreatedAll, ObjectAccessedAll}, b: []EventType{ObjectAccessedAll, ObjectCreatedAll}, }, want: true, }, { name: "not equal", args: args{ a: []EventType{ObjectCreatedAll, ObjectAccessedAll}, b: []EventType{ObjectRemovedAll, ObjectAccessedAll}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := EqualEventTypeList(tt.args.a, tt.args.b); got != tt.want { t.Errorf("EqualEventTypeList() = %v, want %v", got, tt.want) } }) } } func TestEqualFilterRuleList(t *testing.T) { type args struct { a []FilterRule b []FilterRule } tests := []struct { name string args args want bool }{ { name: "same order", args: args{ a: []FilterRule{{Name: "prefix", Value: "prefix1"}, {Name: "suffix", Value: "suffix1"}}, b: []FilterRule{{Name: "prefix", Value: "prefix1"}, {Name: "suffix", Value: "suffix1"}}, }, want: true, }, { name: "different order", args: args{ a: []FilterRule{{Name: "prefix", Value: "prefix1"}, {Name: "suffix", Value: "suffix1"}}, b: []FilterRule{{Name: "suffix", Value: "suffix1"}, {Name: "prefix", Value: "prefix1"}}, }, want: true, }, { name: "not equal", args: args{ a: []FilterRule{{Name: "prefix", Value: "prefix1"}, {Name: "suffix", Value: "suffix1"}}, b: []FilterRule{{Name: "prefix", Value: "prefix2"}, {Name: "suffix", Value: "suffix1"}}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := EqualFilterRuleList(tt.args.a, tt.args.b); got != tt.want { t.Errorf("EqualFilterRuleList() = %v, want %v", got, tt.want) } }) } } func TestConfig_Equal(t *testing.T) { type fields struct { ID string Arn Arn Events []EventType Filter *Filter } type args struct { events []EventType prefix string suffix string } tests := []struct { name string fields fields args args want bool }{ { name: "same order", fields: fields{ Arn: NewArn("minio", "sqs", "", "1", "postgresql"), Events: []EventType{ObjectCreatedAll, ObjectAccessedAll}, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{{Name: "prefix", Value: "prefix1"}, {Name: "suffix", Value: "suffix1"}}, }, }, }, args: args{ events: []EventType{ObjectCreatedAll, ObjectAccessedAll}, prefix: "prefix1", suffix: "suffix1", }, want: true, }, { name: "different order", fields: fields{ Arn: NewArn("minio", "sqs", "", "1", "postgresql"), Events: []EventType{ObjectAccessedAll, ObjectCreatedAll}, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{{Name: "suffix", Value: "suffix1"}, {Name: "prefix", Value: "prefix1"}}, }, }, }, args: args{ events: []EventType{ObjectCreatedAll, ObjectAccessedAll}, prefix: "prefix1", suffix: "suffix1", }, want: true, }, { name: "not equal", fields: fields{ Arn: NewArn("minio", "sqs", "", "1", "postgresql"), Events: []EventType{ObjectAccessedAll}, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{{Name: "suffix", Value: "suffix1"}, {Name: "prefix", Value: "prefix1"}}, }, }, }, args: args{ events: []EventType{ObjectCreatedAll, ObjectAccessedAll}, prefix: "prefix1", suffix: "suffix1", }, want: false, }, { name: "different arn", fields: fields{ Events: []EventType{ObjectAccessedAll}, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{{Name: "suffix", Value: "suffix1"}, {Name: "prefix", Value: "prefix1"}}, }, }, }, args: args{ events: []EventType{ObjectCreatedAll, ObjectAccessedAll}, prefix: "prefix1", suffix: "suffix1", }, want: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { nc := &Config{ ID: tt.fields.ID, Arn: tt.fields.Arn, Events: tt.fields.Events, Filter: tt.fields.Filter, } if got := nc.Equal(tt.args.events, tt.args.prefix, tt.args.suffix); got != tt.want { t.Errorf("Equal() = %v, want %v", got, tt.want) } }) } } func TestConfiguration_RemoveQueueByArnEventsPrefixSuffix(t *testing.T) { type fields struct { XMLName xml.Name LambdaConfigs []LambdaConfig TopicConfigs []TopicConfig QueueConfigs []QueueConfig } type args struct { arn Arn events []EventType prefix string suffix string } tests := []struct { name string fields fields args args wantErr bool }{ { name: "Queue Configuration Removed with events, prefix", fields: fields{ XMLName: xml.Name{}, LambdaConfigs: nil, TopicConfigs: nil, QueueConfigs: []QueueConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "sqs", Region: "", AccountID: "1", Resource: "postgresql", }, Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, }, }, }, }, Queue: "arn:minio:sqs::1:postgresql", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "sqs", Region: "", AccountID: "1", Resource: "postgresql", }, events: []EventType{ ObjectAccessedAll, }, prefix: "x", suffix: "", }, wantErr: false, }, { name: "Queue Configuration Removed with events, prefix, suffix", fields: fields{ XMLName: xml.Name{}, LambdaConfigs: nil, TopicConfigs: nil, QueueConfigs: []QueueConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "sqs", Region: "", AccountID: "1", Resource: "postgresql", }, Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, { Name: "suffix", Value: "y", }, }, }, }, }, Queue: "arn:minio:sqs::1:postgresql", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "sqs", Region: "", AccountID: "1", Resource: "postgresql", }, events: []EventType{ ObjectAccessedAll, }, prefix: "x", suffix: "y", }, wantErr: false, }, { name: "Error Returned Queue Configuration Not Removed", fields: fields{ XMLName: xml.Name{}, LambdaConfigs: nil, TopicConfigs: nil, QueueConfigs: []QueueConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "sqs", Region: "", AccountID: "1", Resource: "postgresql", }, Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, }, }, }, }, Queue: "arn:minio:sqs::1:postgresql", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "sqs", Region: "", AccountID: "1", Resource: "postgresql", }, events: []EventType{ ObjectAccessedAll, }, prefix: "", suffix: "", }, wantErr: true, }, { name: "Queue Configuration Removed with nil Filter", fields: fields{ XMLName: xml.Name{}, LambdaConfigs: nil, TopicConfigs: nil, QueueConfigs: []QueueConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "sqs", Region: "", AccountID: "1", Resource: "postgresql", }, Events: []EventType{ ObjectAccessedAll, }, }, Queue: "arn:minio:sqs::1:postgresql", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "sqs", Region: "", AccountID: "1", Resource: "postgresql", }, events: []EventType{ ObjectAccessedAll, }, prefix: "", suffix: "", }, wantErr: false, }, { name: "Queue Configuration not Removed with no prefix and Filter", fields: fields{ XMLName: xml.Name{}, LambdaConfigs: nil, TopicConfigs: nil, QueueConfigs: []QueueConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "sqs", Region: "", AccountID: "1", Resource: "postgresql", }, Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, }, }, }, }, Queue: "arn:minio:sqs::1:postgresql", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "sqs", Region: "", AccountID: "1", Resource: "postgresql", }, events: []EventType{ ObjectAccessedAll, }, prefix: "", suffix: "", }, wantErr: true, }, { name: "Queue Configuration not Removed with prefix and nil Filter", fields: fields{ XMLName: xml.Name{}, LambdaConfigs: nil, TopicConfigs: nil, QueueConfigs: []QueueConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "sqs", Region: "", AccountID: "1", Resource: "postgresql", }, Events: []EventType{ ObjectAccessedAll, }, }, Queue: "arn:minio:sqs::1:postgresql", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "sqs", Region: "", AccountID: "1", Resource: "postgresql", }, events: []EventType{ ObjectAccessedAll, }, prefix: "x", suffix: "", }, wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { b := &Configuration{ XMLName: tt.fields.XMLName, LambdaConfigs: tt.fields.LambdaConfigs, TopicConfigs: tt.fields.TopicConfigs, QueueConfigs: tt.fields.QueueConfigs, } if err := b.RemoveQueueByArnEventsPrefixSuffix(tt.args.arn, tt.args.events, tt.args.prefix, tt.args.suffix); (err != nil) != tt.wantErr { t.Errorf("RemoveQueueByArnEventsPrefixSuffix() error = %v, wantErr %v", err, tt.wantErr) } }) } } func TestConfiguration_RemoveLambdaByArnEventsPrefixSuffix(t *testing.T) { type fields struct { XMLName xml.Name LambdaConfigs []LambdaConfig TopicConfigs []TopicConfig QueueConfigs []QueueConfig } type args struct { arn Arn events []EventType prefix string suffix string } tests := []struct { name string fields fields args args wantErr bool }{ { name: "Lambda Configuration Removed with events, prefix", fields: fields{ XMLName: xml.Name{}, QueueConfigs: nil, TopicConfigs: nil, LambdaConfigs: []LambdaConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "lambda", Region: "", AccountID: "1", Resource: "provider", }, Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, }, }, }, }, Lambda: "arn:minio:lambda::1:provider", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "lambda", Region: "", AccountID: "1", Resource: "provider", }, events: []EventType{ ObjectAccessedAll, }, prefix: "x", suffix: "", }, wantErr: false, }, { name: "Lambda Configuration Removed with events, prefix, suffix", fields: fields{ XMLName: xml.Name{}, QueueConfigs: nil, TopicConfigs: nil, LambdaConfigs: []LambdaConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "lambda", Region: "", AccountID: "1", Resource: "provider", }, Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, { Name: "suffix", Value: "y", }, }, }, }, }, Lambda: "arn:minio:lambda::1:provider", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "lambda", Region: "", AccountID: "1", Resource: "provider", }, events: []EventType{ ObjectAccessedAll, }, prefix: "x", suffix: "y", }, wantErr: false, }, { name: "Error Returned Lambda Configuration Not Removed", fields: fields{ XMLName: xml.Name{}, QueueConfigs: nil, TopicConfigs: nil, LambdaConfigs: []LambdaConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "lambda", Region: "", AccountID: "1", Resource: "provider", }, Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, }, }, }, }, Lambda: "arn:minio:lambda::1:provider", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "lambda", Region: "", AccountID: "1", Resource: "provider", }, events: []EventType{ ObjectAccessedAll, }, prefix: "", suffix: "", }, wantErr: true, }, { name: "Error Returned Invalid ARN", fields: fields{ XMLName: xml.Name{}, QueueConfigs: nil, TopicConfigs: nil, LambdaConfigs: []LambdaConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "lambda", Region: "", AccountID: "1", Resource: "provider", }, Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, }, }, }, }, Lambda: "arn:minio:lambda::1:provider", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "lambda", Region: "", AccountID: "2", Resource: "provider", }, events: []EventType{ ObjectAccessedAll, }, prefix: "", suffix: "", }, wantErr: true, }, { name: "Lambda Configuration Removed with nil Filter", fields: fields{ XMLName: xml.Name{}, QueueConfigs: nil, TopicConfigs: nil, LambdaConfigs: []LambdaConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "lambda", Region: "", AccountID: "1", Resource: "provider", }, Events: []EventType{ ObjectAccessedAll, }, }, Lambda: "arn:minio:lambda::1:provider", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "lambda", Region: "", AccountID: "1", Resource: "provider", }, events: []EventType{ ObjectAccessedAll, }, prefix: "", suffix: "", }, wantErr: false, }, { name: "Lambda Configuration Not Removed with no prefix and Filter", fields: fields{ XMLName: xml.Name{}, QueueConfigs: nil, TopicConfigs: nil, LambdaConfigs: []LambdaConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "lambda", Region: "", AccountID: "1", Resource: "provider", }, Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, }, }, }, }, Lambda: "arn:minio:lambda::1:provider", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "lambda", Region: "", AccountID: "1", Resource: "provider", }, events: []EventType{ ObjectAccessedAll, }, prefix: "", suffix: "", }, wantErr: true, }, { name: "Lambda Configuration Not Removed with prefix and nil Filter", fields: fields{ XMLName: xml.Name{}, QueueConfigs: nil, TopicConfigs: nil, LambdaConfigs: []LambdaConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "lambda", Region: "", AccountID: "1", Resource: "provider", }, Events: []EventType{ ObjectAccessedAll, }, }, Lambda: "arn:minio:lambda::1:provider", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "lambda", Region: "", AccountID: "1", Resource: "provider", }, events: []EventType{ ObjectAccessedAll, }, prefix: "x", suffix: "", }, wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { b := &Configuration{ XMLName: tt.fields.XMLName, LambdaConfigs: tt.fields.LambdaConfigs, TopicConfigs: tt.fields.TopicConfigs, QueueConfigs: tt.fields.QueueConfigs, } if err := b.RemoveLambdaByArnEventsPrefixSuffix(tt.args.arn, tt.args.events, tt.args.prefix, tt.args.suffix); (err != nil) != tt.wantErr { t.Errorf("RemoveLambdaByArnEventsPrefixSuffix() error = %v, wantErr %v", err, tt.wantErr) } }) } } func TestConfiguration_RemoveTopicByArnEventsPrefixSuffix(t *testing.T) { type fields struct { XMLName xml.Name LambdaConfigs []LambdaConfig TopicConfigs []TopicConfig QueueConfigs []QueueConfig } type args struct { arn Arn events []EventType prefix string suffix string } tests := []struct { name string fields fields args args wantErr bool }{ { name: "Topic Configuration Removed with events, prefix", fields: fields{ XMLName: xml.Name{}, QueueConfigs: nil, LambdaConfigs: nil, TopicConfigs: []TopicConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "sns", Region: "", AccountID: "1", Resource: "kafka", }, Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, }, }, }, }, Topic: "arn:minio:sns::1:kafka", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "sns", Region: "", AccountID: "1", Resource: "kafka", }, events: []EventType{ ObjectAccessedAll, }, prefix: "x", suffix: "", }, wantErr: false, }, { name: "Topic Configuration Removed with events, prefix, suffix", fields: fields{ XMLName: xml.Name{}, QueueConfigs: nil, LambdaConfigs: nil, TopicConfigs: []TopicConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "sns", Region: "", AccountID: "1", Resource: "kafka", }, Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, { Name: "suffix", Value: "y", }, }, }, }, }, Topic: "arn:minio:sns::1:kafka", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "sns", Region: "", AccountID: "1", Resource: "kafka", }, events: []EventType{ ObjectAccessedAll, }, prefix: "x", suffix: "y", }, wantErr: false, }, { name: "Error Returned Topic Configuration Not Removed", fields: fields{ XMLName: xml.Name{}, QueueConfigs: nil, LambdaConfigs: nil, TopicConfigs: []TopicConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "sns", Region: "", AccountID: "1", Resource: "kafka", }, Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, }, }, }, }, Topic: "arn:minio:sns::1:kafka", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "sns", Region: "", AccountID: "1", Resource: "kafka", }, events: []EventType{ ObjectAccessedAll, }, prefix: "", suffix: "", }, wantErr: true, }, { name: "Topic Configuration Removed with nil Filter", fields: fields{ XMLName: xml.Name{}, QueueConfigs: nil, LambdaConfigs: nil, TopicConfigs: []TopicConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "sns", Region: "", AccountID: "1", Resource: "kafka", }, Events: []EventType{ ObjectAccessedAll, }, }, Topic: "arn:minio:sns::1:kafka", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "sns", Region: "", AccountID: "1", Resource: "kafka", }, events: []EventType{ ObjectAccessedAll, }, prefix: "", suffix: "", }, wantErr: false, }, { name: "Topic Configuration Not Removed, prefix empty", fields: fields{ XMLName: xml.Name{}, QueueConfigs: nil, LambdaConfigs: nil, TopicConfigs: []TopicConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "sns", Region: "", AccountID: "1", Resource: "kafka", }, Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, }, }, }, }, Topic: "arn:minio:sns::1:kafka", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "sns", Region: "", AccountID: "1", Resource: "kafka", }, events: []EventType{ ObjectAccessedAll, }, prefix: "", suffix: "", }, wantErr: true, }, { name: "Topic Configuration Not Removed, Config Events Empty", fields: fields{ XMLName: xml.Name{}, QueueConfigs: nil, LambdaConfigs: nil, TopicConfigs: []TopicConfig{ { Config: Config{ ID: "", Arn: Arn{ Partition: "minio", Service: "sns", Region: "", AccountID: "1", Resource: "kafka", }, Events: []EventType{ ObjectAccessedAll, }, }, Topic: "arn:minio:sns::1:kafka", }, }, }, args: args{ arn: Arn{ Partition: "minio", Service: "sns", Region: "", AccountID: "1", Resource: "kafka", }, events: []EventType{ ObjectAccessedAll, }, prefix: "x", suffix: "", }, wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { b := &Configuration{ XMLName: tt.fields.XMLName, LambdaConfigs: tt.fields.LambdaConfigs, TopicConfigs: tt.fields.TopicConfigs, QueueConfigs: tt.fields.QueueConfigs, } if err := b.RemoveTopicByArnEventsPrefixSuffix(tt.args.arn, tt.args.events, tt.args.prefix, tt.args.suffix); (err != nil) != tt.wantErr { t.Errorf("RemoveTopicByArnEventsPrefixSuffix() error = %v, wantErr %v", err, tt.wantErr) } }) } } func TestConfigEqual(t *testing.T) { type args struct { config Config events []EventType prefix string suffix string } tests := []struct { name string args args expected bool }{ { name: "Config equal true", args: args{ events: []EventType{ObjectAccessedAll}, prefix: "x", suffix: "", config: Config{ ID: "", Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, }, }, }, }, }, expected: true, }, { name: "Config Events different", args: args{ events: []EventType{ObjectCreatedAll}, prefix: "x", suffix: "", config: Config{ ID: "", Events: []EventType{ ObjectAccessedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, }, }, }, }, }, expected: false, }, { name: "Config prefix, events and suffix equal", args: args{ events: []EventType{ObjectCreatedAll}, prefix: "x", suffix: "s", config: Config{ ID: "", Events: []EventType{ ObjectCreatedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, { Name: "suffix", Value: "s", }, }, }, }, }, }, expected: true, }, { name: "Config.Filter nil, filter prefix and suffix empty", args: args{ events: []EventType{ObjectCreatedAll}, prefix: "", suffix: "", config: Config{ ID: "", Events: []EventType{ ObjectCreatedAll, }, }, }, expected: true, }, { name: "Config.Filter nil and events, prefix and suffix empty", args: args{ events: []EventType{}, prefix: "", suffix: "", config: Config{}, }, expected: true, }, { name: "Config prefix empty, Config.Filters not nil", args: args{ events: []EventType{ObjectCreatedAll}, prefix: "", suffix: "", config: Config{ ID: "", Events: []EventType{ ObjectCreatedAll, }, Filter: &Filter{ S3Key: S3Key{ FilterRules: []FilterRule{ { Name: "prefix", Value: "x", }, }, }, }, }, }, expected: false, }, { name: "Config prefix not empty, Config.Filters nil", args: args{ events: []EventType{ObjectCreatedAll}, prefix: "x", suffix: "", config: Config{ ID: "", Events: []EventType{ ObjectCreatedAll, }, }, }, expected: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if res := tt.args.config.Equal(tt.args.events, tt.args.prefix, tt.args.suffix); res != tt.expected { t.Errorf("config.Equal() = %v, expected %v", res, tt.expected) } }) } } func TestNewArnFromString(t *testing.T) { t.Run("valid ARN", func(t *testing.T) { arn := NewArn("partition", "service", "region", "accountID", "resource") arnString := arn.String() arnFromString, err := NewArnFromString(arnString) if err != nil { t.Fatalf("did not exect an error, but got %v", err) } if arnFromString.String() != arnString { t.Errorf("expected ARNs to be equal, but they are not: arnFromString = %s, arn = %s", arnFromString.String(), arnString) } }) t.Run("invalid ARN format", func(t *testing.T) { _, err := NewArnFromString("arn:only:four:parts") if err != ErrInvalidArnFormat { t.Errorf("expected an error %v, but got %v", ErrInvalidArnFormat, err) } }) t.Run("invalid ARN prefix", func(t *testing.T) { _, err := NewArnFromString("non-arn:partition:service:region:accountID:resource") if err != ErrInvalidArnPrefix { t.Errorf("expected an error %v, but got %v", ErrInvalidArnPrefix, err) } }) } minio-go-7.0.97/pkg/policy/000077500000000000000000000000001510244170000154065ustar00rootroot00000000000000minio-go-7.0.97/pkg/policy/bucket-policy-condition.go000066400000000000000000000061141510244170000224750ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package policy import ( "github.com/minio/minio-go/v7/pkg/set" ) // ConditionKeyMap - map of policy condition key and value. type ConditionKeyMap map[string]set.StringSet // Add - adds key and value. The value is appended If key already exists. func (ckm ConditionKeyMap) Add(key string, value set.StringSet) { if v, ok := ckm[key]; ok { ckm[key] = v.Union(value) } else { ckm[key] = set.CopyStringSet(value) } } // Remove - removes value of given key. If key has empty after removal, the key is also removed. func (ckm ConditionKeyMap) Remove(key string, value set.StringSet) { if v, ok := ckm[key]; ok { if value != nil { ckm[key] = v.Difference(value) } if ckm[key].IsEmpty() { delete(ckm, key) } } } // RemoveKey - removes key and its value. func (ckm ConditionKeyMap) RemoveKey(key string) { delete(ckm, key) } // CopyConditionKeyMap - returns new copy of given ConditionKeyMap. func CopyConditionKeyMap(condKeyMap ConditionKeyMap) ConditionKeyMap { out := make(ConditionKeyMap) for k, v := range condKeyMap { out[k] = set.CopyStringSet(v) } return out } // mergeConditionKeyMap - returns a new ConditionKeyMap which contains merged key/value of given two ConditionKeyMap. func mergeConditionKeyMap(condKeyMap1, condKeyMap2 ConditionKeyMap) ConditionKeyMap { out := CopyConditionKeyMap(condKeyMap1) for k, v := range condKeyMap2 { if ev, ok := out[k]; ok { out[k] = ev.Union(v) } else { out[k] = set.CopyStringSet(v) } } return out } // ConditionMap - map of condition and conditional values. type ConditionMap map[string]ConditionKeyMap // Add - adds condition key and condition value. The value is appended if key already exists. func (cond ConditionMap) Add(condKey string, condKeyMap ConditionKeyMap) { if v, ok := cond[condKey]; ok { cond[condKey] = mergeConditionKeyMap(v, condKeyMap) } else { cond[condKey] = CopyConditionKeyMap(condKeyMap) } } // Remove - removes condition key and its value. func (cond ConditionMap) Remove(condKey string) { delete(cond, condKey) } // mergeConditionMap - returns new ConditionMap which contains merged key/value of two ConditionMap. func mergeConditionMap(condMap1, condMap2 ConditionMap) ConditionMap { out := make(ConditionMap) for k, v := range condMap1 { out[k] = CopyConditionKeyMap(v) } for k, v := range condMap2 { if ev, ok := out[k]; ok { out[k] = mergeConditionKeyMap(ev, v) } else { out[k] = CopyConditionKeyMap(v) } } return out } minio-go-7.0.97/pkg/policy/bucket-policy-condition_test.go000066400000000000000000000231361510244170000235370ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package policy import ( "encoding/json" "testing" "github.com/minio/minio-go/v7/pkg/set" ) // ConditionKeyMap.Add() is called and the result is validated. func TestConditionKeyMapAdd(t *testing.T) { condKeyMap := make(ConditionKeyMap) testCases := []struct { key string value set.StringSet expectedResult string }{ // Add new key and value. {"s3:prefix", set.CreateStringSet("hello"), `{"s3:prefix":["hello"]}`}, // Add existing key and value. {"s3:prefix", set.CreateStringSet("hello"), `{"s3:prefix":["hello"]}`}, // Add existing key and not value. {"s3:prefix", set.CreateStringSet("world"), `{"s3:prefix":["hello","world"]}`}, } for _, testCase := range testCases { condKeyMap.Add(testCase.key, testCase.value) if data, err := json.Marshal(condKeyMap); err != nil { t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) } else { if string(data) != testCase.expectedResult { t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) } } } } // ConditionKeyMap.Remove() is called and the result is validated. func TestConditionKeyMapRemove(t *testing.T) { condKeyMap := make(ConditionKeyMap) condKeyMap.Add("s3:prefix", set.CreateStringSet("hello", "world")) testCases := []struct { key string value set.StringSet expectedResult string }{ // Remove non-existent key and value. {"s3:myprefix", set.CreateStringSet("hello"), `{"s3:prefix":["hello","world"]}`}, // Remove existing key and value. {"s3:prefix", set.CreateStringSet("hello"), `{"s3:prefix":["world"]}`}, // Remove existing key to make the key also removed. {"s3:prefix", set.CreateStringSet("world"), `{}`}, } for _, testCase := range testCases { condKeyMap.Remove(testCase.key, testCase.value) if data, err := json.Marshal(condKeyMap); err != nil { t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) } else { if string(data) != testCase.expectedResult { t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) } } } } // ConditionKeyMap.RemoveKey() is called and the result is validated. func TestConditionKeyMapRemoveKey(t *testing.T) { condKeyMap := make(ConditionKeyMap) condKeyMap.Add("s3:prefix", set.CreateStringSet("hello", "world")) testCases := []struct { key string expectedResult string }{ // Remove non-existent key. {"s3:myprefix", `{"s3:prefix":["hello","world"]}`}, // Remove existing key. {"s3:prefix", `{}`}, } for _, testCase := range testCases { condKeyMap.RemoveKey(testCase.key) if data, err := json.Marshal(condKeyMap); err != nil { t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) } else { if string(data) != testCase.expectedResult { t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) } } } } // CopyConditionKeyMap() is called and the result is validated. func TestCopyConditionKeyMap(t *testing.T) { emptyCondKeyMap := make(ConditionKeyMap) nonEmptyCondKeyMap := make(ConditionKeyMap) nonEmptyCondKeyMap.Add("s3:prefix", set.CreateStringSet("hello", "world")) testCases := []struct { condKeyMap ConditionKeyMap expectedResult string }{ // To test empty ConditionKeyMap. {emptyCondKeyMap, `{}`}, // To test non-empty ConditionKeyMap. {nonEmptyCondKeyMap, `{"s3:prefix":["hello","world"]}`}, } for _, testCase := range testCases { condKeyMap := CopyConditionKeyMap(testCase.condKeyMap) if data, err := json.Marshal(condKeyMap); err != nil { t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) } else { if string(data) != testCase.expectedResult { t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) } } } } // mergeConditionKeyMap() is called and the result is validated. func TestMergeConditionKeyMap(t *testing.T) { condKeyMap1 := make(ConditionKeyMap) condKeyMap1.Add("s3:prefix", set.CreateStringSet("hello")) condKeyMap2 := make(ConditionKeyMap) condKeyMap2.Add("s3:prefix", set.CreateStringSet("world")) condKeyMap3 := make(ConditionKeyMap) condKeyMap3.Add("s3:myprefix", set.CreateStringSet("world")) testCases := []struct { condKeyMap1 ConditionKeyMap condKeyMap2 ConditionKeyMap expectedResult string }{ // Both arguments are empty. {make(ConditionKeyMap), make(ConditionKeyMap), `{}`}, // First argument is empty. {make(ConditionKeyMap), condKeyMap1, `{"s3:prefix":["hello"]}`}, // Second argument is empty. {condKeyMap1, make(ConditionKeyMap), `{"s3:prefix":["hello"]}`}, // Both arguments are same value. {condKeyMap1, condKeyMap1, `{"s3:prefix":["hello"]}`}, // Value of second argument will be merged. {condKeyMap1, condKeyMap2, `{"s3:prefix":["hello","world"]}`}, // second argument will be added. {condKeyMap1, condKeyMap3, `{"s3:myprefix":["world"],"s3:prefix":["hello"]}`}, } for _, testCase := range testCases { condKeyMap := mergeConditionKeyMap(testCase.condKeyMap1, testCase.condKeyMap2) if data, err := json.Marshal(condKeyMap); err != nil { t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) } else { if string(data) != testCase.expectedResult { t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) } } } } // ConditionMap.Add() is called and the result is validated. func TestConditionMapAdd(t *testing.T) { condMap := make(ConditionMap) condKeyMap1 := make(ConditionKeyMap) condKeyMap1.Add("s3:prefix", set.CreateStringSet("hello")) condKeyMap2 := make(ConditionKeyMap) condKeyMap2.Add("s3:prefix", set.CreateStringSet("hello", "world")) testCases := []struct { key string value ConditionKeyMap expectedResult string }{ // Add new key and value. {"StringEquals", condKeyMap1, `{"StringEquals":{"s3:prefix":["hello"]}}`}, // Add existing key and value. {"StringEquals", condKeyMap1, `{"StringEquals":{"s3:prefix":["hello"]}}`}, // Add existing key and not value. {"StringEquals", condKeyMap2, `{"StringEquals":{"s3:prefix":["hello","world"]}}`}, } for _, testCase := range testCases { condMap.Add(testCase.key, testCase.value) if data, err := json.Marshal(condMap); err != nil { t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) } else { if string(data) != testCase.expectedResult { t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) } } } } // ConditionMap.Remove() is called and the result is validated. func TestConditionMapRemove(t *testing.T) { condMap := make(ConditionMap) condKeyMap := make(ConditionKeyMap) condKeyMap.Add("s3:prefix", set.CreateStringSet("hello", "world")) condMap.Add("StringEquals", condKeyMap) testCases := []struct { key string expectedResult string }{ // Remove non-existent key. {"StringNotEquals", `{"StringEquals":{"s3:prefix":["hello","world"]}}`}, // Remove existing key. {"StringEquals", `{}`}, } for _, testCase := range testCases { condMap.Remove(testCase.key) if data, err := json.Marshal(condMap); err != nil { t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) } else { if string(data) != testCase.expectedResult { t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) } } } } // mergeConditionMap() is called and the result is validated. func TestMergeConditionMap(t *testing.T) { condKeyMap1 := make(ConditionKeyMap) condKeyMap1.Add("s3:prefix", set.CreateStringSet("hello")) condMap1 := make(ConditionMap) condMap1.Add("StringEquals", condKeyMap1) condKeyMap2 := make(ConditionKeyMap) condKeyMap2.Add("s3:prefix", set.CreateStringSet("world")) condMap2 := make(ConditionMap) condMap2.Add("StringEquals", condKeyMap2) condMap3 := make(ConditionMap) condMap3.Add("StringNotEquals", condKeyMap2) testCases := []struct { condMap1 ConditionMap condMap2 ConditionMap expectedResult string }{ // Both arguments are empty. {make(ConditionMap), make(ConditionMap), `{}`}, // First argument is empty. {make(ConditionMap), condMap1, `{"StringEquals":{"s3:prefix":["hello"]}}`}, // Second argument is empty. {condMap1, make(ConditionMap), `{"StringEquals":{"s3:prefix":["hello"]}}`}, // Both arguments are same value. {condMap1, condMap1, `{"StringEquals":{"s3:prefix":["hello"]}}`}, // Value of second argument will be merged. {condMap1, condMap2, `{"StringEquals":{"s3:prefix":["hello","world"]}}`}, // second argument will be added. {condMap1, condMap3, `{"StringEquals":{"s3:prefix":["hello"]},"StringNotEquals":{"s3:prefix":["world"]}}`}, } for _, testCase := range testCases { condMap := mergeConditionMap(testCase.condMap1, testCase.condMap2) if data, err := json.Marshal(condMap); err != nil { t.Fatalf("Unable to marshal ConditionKeyMap to JSON, %s", err) } else { if string(data) != testCase.expectedResult { t.Fatalf("case: %+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) } } } } minio-go-7.0.97/pkg/policy/bucket-policy.go000066400000000000000000000506631510244170000205210ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package policy import ( "encoding/json" "errors" "reflect" "strings" "github.com/minio/minio-go/v7/pkg/set" ) // BucketPolicy - Bucket level policy. type BucketPolicy string // Different types of Policies currently supported for buckets. const ( BucketPolicyNone BucketPolicy = "none" BucketPolicyReadOnly BucketPolicy = "readonly" BucketPolicyReadWrite BucketPolicy = "readwrite" BucketPolicyWriteOnly BucketPolicy = "writeonly" ) // IsValidBucketPolicy - returns true if policy is valid and supported, false otherwise. func (p BucketPolicy) IsValidBucketPolicy() bool { switch p { case BucketPolicyNone, BucketPolicyReadOnly, BucketPolicyReadWrite, BucketPolicyWriteOnly: return true } return false } // Resource prefix for all aws resources. const awsResourcePrefix = "arn:aws:s3:::" // Common bucket actions for both read and write policies. var commonBucketActions = set.CreateStringSet("s3:GetBucketLocation") // Read only bucket actions. var readOnlyBucketActions = set.CreateStringSet("s3:ListBucket") // Write only bucket actions. var writeOnlyBucketActions = set.CreateStringSet("s3:ListBucketMultipartUploads") // Read only object actions. var readOnlyObjectActions = set.CreateStringSet("s3:GetObject") // Write only object actions. var writeOnlyObjectActions = set.CreateStringSet("s3:AbortMultipartUpload", "s3:DeleteObject", "s3:ListMultipartUploadParts", "s3:PutObject") // Read and write object actions. var readWriteObjectActions = readOnlyObjectActions.Union(writeOnlyObjectActions) // All valid bucket and object actions. var validActions = commonBucketActions. Union(readOnlyBucketActions). Union(writeOnlyBucketActions). Union(readOnlyObjectActions). Union(writeOnlyObjectActions) var startsWithFunc = func(resource, resourcePrefix string) bool { return strings.HasPrefix(resource, resourcePrefix) } // User - canonical users list. type User struct { AWS set.StringSet `json:"AWS,omitempty"` CanonicalUser set.StringSet `json:"CanonicalUser,omitempty"` } // UnmarshalJSON is a custom json unmarshaler for Principal field, // the reason is that Principal can take a json struct represented by // User string but it can also take a string. func (u *User) UnmarshalJSON(data []byte) error { // Try to unmarshal data in a struct equal to User, // to avoid infinite recursive call of this function type AliasUser User var au AliasUser err := json.Unmarshal(data, &au) if err == nil { *u = User(au) return nil } // Data type is not known, check if it is a json string // which contains a star, which is permitted in the spec var str string err = json.Unmarshal(data, &str) if err == nil { if str != "*" { return errors.New("unrecognized Principal field") } *u = User{AWS: set.CreateStringSet("*")} return nil } return err } // Statement - minio policy statement type Statement struct { Actions set.StringSet `json:"Action"` Conditions ConditionMap `json:"Condition,omitempty"` Effect string Principal User `json:"Principal"` Resources set.StringSet `json:"Resource"` Sid string } // BucketAccessPolicy - minio policy collection type BucketAccessPolicy struct { Version string // date in YYYY-MM-DD format Statements []Statement `json:"Statement"` } // isValidStatement - returns whether given statement is valid to process for given bucket name. func isValidStatement(statement Statement, bucketName string) bool { if statement.Actions.Intersection(validActions).IsEmpty() { return false } if statement.Effect != "Allow" { return false } if statement.Principal.AWS == nil || !statement.Principal.AWS.Contains("*") { return false } bucketResource := awsResourcePrefix + bucketName if statement.Resources.Contains(bucketResource) { return true } if statement.Resources.FuncMatch(startsWithFunc, bucketResource+"/").IsEmpty() { return false } return true } // Returns new statements with bucket actions for given policy. func newBucketStatement(policy BucketPolicy, bucketName, prefix string) (statements []Statement) { statements = []Statement{} if policy == BucketPolicyNone || bucketName == "" { return statements } bucketResource := set.CreateStringSet(awsResourcePrefix + bucketName) statement := Statement{ Actions: commonBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: bucketResource, Sid: "", } statements = append(statements, statement) if policy == BucketPolicyReadOnly || policy == BucketPolicyReadWrite { statement = Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: bucketResource, Sid: "", } if prefix != "" { condKeyMap := make(ConditionKeyMap) condKeyMap.Add("s3:prefix", set.CreateStringSet(prefix+"*")) condMap := make(ConditionMap) condMap.Add("StringLike", condKeyMap) statement.Conditions = condMap } statements = append(statements, statement) } if policy == BucketPolicyWriteOnly || policy == BucketPolicyReadWrite { statement = Statement{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: bucketResource, Sid: "", } statements = append(statements, statement) } return statements } // Returns new statements contains object actions for given policy. func newObjectStatement(policy BucketPolicy, bucketName, prefix string) (statements []Statement) { statements = []Statement{} if policy == BucketPolicyNone || bucketName == "" { return statements } statement := Statement{ Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet(awsResourcePrefix + bucketName + "/" + prefix + "*"), Sid: "", } switch policy { case BucketPolicyReadOnly: statement.Actions = readOnlyObjectActions case BucketPolicyWriteOnly: statement.Actions = writeOnlyObjectActions case BucketPolicyReadWrite: statement.Actions = readWriteObjectActions } statements = append(statements, statement) return statements } // Returns new statements for given policy, bucket and prefix. func newStatements(policy BucketPolicy, bucketName, prefix string) (statements []Statement) { statements = []Statement{} ns := newBucketStatement(policy, bucketName, prefix) statements = append(statements, ns...) ns = newObjectStatement(policy, bucketName, prefix) statements = append(statements, ns...) return statements } // Returns whether given bucket statements are used by other than given prefix statements. func getInUsePolicy(statements []Statement, bucketName, prefix string) (readOnlyInUse, writeOnlyInUse bool) { resourcePrefix := awsResourcePrefix + bucketName + "/" objectResource := awsResourcePrefix + bucketName + "/" + prefix + "*" for _, s := range statements { if !s.Resources.Contains(objectResource) && !s.Resources.FuncMatch(startsWithFunc, resourcePrefix).IsEmpty() { if s.Actions.Intersection(readOnlyObjectActions).Equals(readOnlyObjectActions) { readOnlyInUse = true } if s.Actions.Intersection(writeOnlyObjectActions).Equals(writeOnlyObjectActions) { writeOnlyInUse = true } } if readOnlyInUse && writeOnlyInUse { break } } return readOnlyInUse, writeOnlyInUse } // Removes object actions in given statement. func removeObjectActions(statement Statement, objectResource string) Statement { if statement.Conditions == nil { if len(statement.Resources) > 1 { statement.Resources.Remove(objectResource) } else { statement.Actions = statement.Actions.Difference(readOnlyObjectActions) statement.Actions = statement.Actions.Difference(writeOnlyObjectActions) } } return statement } // Removes bucket actions for given policy in given statement. func removeBucketActions(statement Statement, prefix, bucketResource string, readOnlyInUse, writeOnlyInUse bool) Statement { removeReadOnly := func() { if !statement.Actions.Intersection(readOnlyBucketActions).Equals(readOnlyBucketActions) { return } if statement.Conditions == nil { statement.Actions = statement.Actions.Difference(readOnlyBucketActions) return } if prefix != "" { stringEqualsValue := statement.Conditions["StringEquals"] values := set.NewStringSet() if stringEqualsValue != nil { values = stringEqualsValue["s3:prefix"] if values == nil { values = set.NewStringSet() } } values.Remove(prefix) if stringEqualsValue != nil { if values.IsEmpty() { delete(stringEqualsValue, "s3:prefix") } if len(stringEqualsValue) == 0 { delete(statement.Conditions, "StringEquals") } } if len(statement.Conditions) == 0 { statement.Conditions = nil statement.Actions = statement.Actions.Difference(readOnlyBucketActions) } } } removeWriteOnly := func() { if statement.Conditions == nil { statement.Actions = statement.Actions.Difference(writeOnlyBucketActions) } } if len(statement.Resources) > 1 { statement.Resources.Remove(bucketResource) } else { if !readOnlyInUse { removeReadOnly() } if !writeOnlyInUse { removeWriteOnly() } } return statement } // Returns statements containing removed actions/statements for given // policy, bucket name and prefix. func removeStatements(statements []Statement, bucketName, prefix string) []Statement { bucketResource := awsResourcePrefix + bucketName objectResource := awsResourcePrefix + bucketName + "/" + prefix + "*" readOnlyInUse, writeOnlyInUse := getInUsePolicy(statements, bucketName, prefix) out := []Statement{} readOnlyBucketStatements := []Statement{} s3PrefixValues := set.NewStringSet() for _, statement := range statements { if !isValidStatement(statement, bucketName) { out = append(out, statement) continue } if statement.Resources.Contains(bucketResource) { if statement.Conditions != nil { statement = removeBucketActions(statement, prefix, bucketResource, false, false) } else { statement = removeBucketActions(statement, prefix, bucketResource, readOnlyInUse, writeOnlyInUse) } } else if statement.Resources.Contains(objectResource) { statement = removeObjectActions(statement, objectResource) } if !statement.Actions.IsEmpty() { if statement.Resources.Contains(bucketResource) && statement.Actions.Intersection(readOnlyBucketActions).Equals(readOnlyBucketActions) && statement.Effect == "Allow" && statement.Principal.AWS.Contains("*") { if statement.Conditions != nil { stringEqualsValue := statement.Conditions["StringEquals"] values := set.NewStringSet() if stringEqualsValue != nil { values = stringEqualsValue["s3:prefix"] if values == nil { values = set.NewStringSet() } } s3PrefixValues = s3PrefixValues.Union(values.ApplyFunc(func(v string) string { return bucketResource + "/" + v + "*" })) } else if !s3PrefixValues.IsEmpty() { readOnlyBucketStatements = append(readOnlyBucketStatements, statement) continue } } out = append(out, statement) } } skipBucketStatement := true resourcePrefix := awsResourcePrefix + bucketName + "/" for _, statement := range out { if !statement.Resources.FuncMatch(startsWithFunc, resourcePrefix).IsEmpty() && s3PrefixValues.Intersection(statement.Resources).IsEmpty() { skipBucketStatement = false break } } for _, statement := range readOnlyBucketStatements { if skipBucketStatement && statement.Resources.Contains(bucketResource) && statement.Effect == "Allow" && statement.Principal.AWS.Contains("*") && statement.Conditions == nil { continue } out = append(out, statement) } if len(out) == 1 { statement := out[0] if statement.Resources.Contains(bucketResource) && statement.Actions.Intersection(commonBucketActions).Equals(commonBucketActions) && statement.Effect == "Allow" && statement.Principal.AWS.Contains("*") && statement.Conditions == nil { out = []Statement{} } } return out } // Appends given statement into statement list to have unique statements. // - If statement already exists in statement list, it ignores. // - If statement exists with different conditions, they are merged. // - Else the statement is appended to statement list. func appendStatement(statements []Statement, statement Statement) []Statement { for i, s := range statements { if s.Actions.Equals(statement.Actions) && s.Effect == statement.Effect && s.Principal.AWS.Equals(statement.Principal.AWS) && reflect.DeepEqual(s.Conditions, statement.Conditions) { statements[i].Resources = s.Resources.Union(statement.Resources) return statements } else if s.Resources.Equals(statement.Resources) && s.Effect == statement.Effect && s.Principal.AWS.Equals(statement.Principal.AWS) && reflect.DeepEqual(s.Conditions, statement.Conditions) { statements[i].Actions = s.Actions.Union(statement.Actions) return statements } if s.Resources.Intersection(statement.Resources).Equals(statement.Resources) && s.Actions.Intersection(statement.Actions).Equals(statement.Actions) && s.Effect == statement.Effect && s.Principal.AWS.Intersection(statement.Principal.AWS).Equals(statement.Principal.AWS) { if reflect.DeepEqual(s.Conditions, statement.Conditions) { return statements } if s.Conditions != nil && statement.Conditions != nil { if s.Resources.Equals(statement.Resources) { statements[i].Conditions = mergeConditionMap(s.Conditions, statement.Conditions) return statements } } } } if !statement.Actions.IsEmpty() || !statement.Resources.IsEmpty() { return append(statements, statement) } return statements } // Appends two statement lists. func appendStatements(statements, appendStatements []Statement) []Statement { for _, s := range appendStatements { statements = appendStatement(statements, s) } return statements } // Returns policy of given bucket statement. func getBucketPolicy(statement Statement, prefix string) (commonFound, readOnly, writeOnly bool) { if statement.Effect != "Allow" || !statement.Principal.AWS.Contains("*") { return commonFound, readOnly, writeOnly } if statement.Actions.Intersection(commonBucketActions).Equals(commonBucketActions) && statement.Conditions == nil { commonFound = true } if statement.Actions.Intersection(writeOnlyBucketActions).Equals(writeOnlyBucketActions) && statement.Conditions == nil { writeOnly = true } if statement.Actions.Intersection(readOnlyBucketActions).Equals(readOnlyBucketActions) { if prefix != "" && statement.Conditions != nil { if stringEqualsValue, ok := statement.Conditions["StringEquals"]; ok { if s3PrefixValues, ok := stringEqualsValue["s3:prefix"]; ok { if s3PrefixValues.Contains(prefix) { readOnly = true } } } else if stringNotEqualsValue, ok := statement.Conditions["StringNotEquals"]; ok { if s3PrefixValues, ok := stringNotEqualsValue["s3:prefix"]; ok { if !s3PrefixValues.Contains(prefix) { readOnly = true } } } else if stringLikeValue, ok := statement.Conditions["StringLike"]; ok { if s3PrefixValues, ok := stringLikeValue["s3:prefix"]; ok { if s3PrefixValues.Contains(prefix + "*") { readOnly = true } } } else if stringNotLikeValue, ok := statement.Conditions["StringNotLike"]; ok { if s3PrefixValues, ok := stringNotLikeValue["s3:prefix"]; ok { if !s3PrefixValues.Contains(prefix + "*") { readOnly = true } } } } else if prefix == "" && statement.Conditions == nil { readOnly = true } else if prefix != "" && statement.Conditions == nil { readOnly = true } } return commonFound, readOnly, writeOnly } // Returns policy of given object statement. func getObjectPolicy(statement Statement) (readOnly, writeOnly bool) { if statement.Effect == "Allow" && statement.Principal.AWS.Contains("*") && statement.Conditions == nil { if statement.Actions.Intersection(readOnlyObjectActions).Equals(readOnlyObjectActions) { readOnly = true } if statement.Actions.Intersection(writeOnlyObjectActions).Equals(writeOnlyObjectActions) { writeOnly = true } } return readOnly, writeOnly } // GetPolicy - Returns policy of given bucket name, prefix in given statements. func GetPolicy(statements []Statement, bucketName, prefix string) BucketPolicy { bucketResource := awsResourcePrefix + bucketName objectResource := awsResourcePrefix + bucketName + "/" + prefix + "*" bucketCommonFound := false bucketReadOnly := false bucketWriteOnly := false matchedResource := "" objReadOnly := false objWriteOnly := false for _, s := range statements { matchedObjResources := set.NewStringSet() if s.Resources.Contains(objectResource) { matchedObjResources.Add(objectResource) } else { matchedObjResources = s.Resources.FuncMatch(resourceMatch, objectResource) } if !matchedObjResources.IsEmpty() { readOnly, writeOnly := getObjectPolicy(s) for resource := range matchedObjResources { if len(matchedResource) < len(resource) { objReadOnly = readOnly objWriteOnly = writeOnly matchedResource = resource } else if len(matchedResource) == len(resource) { objReadOnly = objReadOnly || readOnly objWriteOnly = objWriteOnly || writeOnly matchedResource = resource } } } if s.Resources.Contains(bucketResource) { commonFound, readOnly, writeOnly := getBucketPolicy(s, prefix) bucketCommonFound = bucketCommonFound || commonFound bucketReadOnly = bucketReadOnly || readOnly bucketWriteOnly = bucketWriteOnly || writeOnly } } policy := BucketPolicyNone if bucketCommonFound { if bucketReadOnly && bucketWriteOnly && objReadOnly && objWriteOnly { policy = BucketPolicyReadWrite } else if bucketReadOnly && objReadOnly { policy = BucketPolicyReadOnly } else if bucketWriteOnly && objWriteOnly { policy = BucketPolicyWriteOnly } } return policy } // GetPolicies - returns a map of policies of given bucket name, prefix in given statements. func GetPolicies(statements []Statement, bucketName, prefix string) map[string]BucketPolicy { policyRules := map[string]BucketPolicy{} objResources := set.NewStringSet() // Search all resources related to objects policy for _, s := range statements { for r := range s.Resources { if strings.HasPrefix(r, awsResourcePrefix+bucketName+"/"+prefix) { objResources.Add(r) } } } // Pretend that policy resource as an actual object and fetch its policy for r := range objResources { // Put trailing * if exists in asterisk asterisk := "" if strings.HasSuffix(r, "*") { r = r[:len(r)-1] asterisk = "*" } var objectPath string if len(r) >= len(awsResourcePrefix+bucketName)+1 { objectPath = r[len(awsResourcePrefix+bucketName)+1:] } p := GetPolicy(statements, bucketName, objectPath) policyRules[bucketName+"/"+objectPath+asterisk] = p } return policyRules } // SetPolicy - Returns new statements containing policy of given bucket name and prefix are appended. func SetPolicy(statements []Statement, policy BucketPolicy, bucketName, prefix string) []Statement { out := removeStatements(statements, bucketName, prefix) // fmt.Println("out = ") // printstatement(out) ns := newStatements(policy, bucketName, prefix) // fmt.Println("ns = ") // printstatement(ns) rv := appendStatements(out, ns) // fmt.Println("rv = ") // printstatement(rv) return rv } // Match function matches wild cards in 'pattern' for resource. func resourceMatch(pattern, resource string) bool { if pattern == "" { return resource == pattern } if pattern == "*" { return true } parts := strings.Split(pattern, "*") if len(parts) == 1 { return resource == pattern } tGlob := strings.HasSuffix(pattern, "*") end := len(parts) - 1 if !strings.HasPrefix(resource, parts[0]) { return false } for i := 1; i < end; i++ { if !strings.Contains(resource, parts[i]) { return false } idx := strings.Index(resource, parts[i]) + len(parts[i]) resource = resource[idx:] } return tGlob || strings.HasSuffix(resource, parts[end]) } minio-go-7.0.97/pkg/policy/bucket-policy_test.go000066400000000000000000003161601510244170000215550ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package policy import ( "encoding/json" "fmt" "reflect" "testing" "github.com/minio/minio-go/v7/pkg/set" ) // TestUnmarshalBucketPolicy tests unmarsheling various examples // of bucket policies, to verify the correctness of BucketAccessPolicy // struct defined in this package. func TestUnmarshalBucketPolicy(t *testing.T) { testCases := []struct { policyData string shouldSucceed bool }{ // Test 1 {policyData: `{ "Version":"2012-10-17", "Statement":[ { "Sid":"AddCannedAcl", "Effect":"Allow", "Principal": {"AWS": ["arn:aws:iam::111122223333:root","arn:aws:iam::444455556666:root"]}, "Action":["s3:PutObject","s3:PutObjectAcl"], "Resource":["arn:aws:s3:::examplebucket/*"], "Condition":{"StringEquals":{"s3:x-amz-acl":["public-read"]}} } ] }`, shouldSucceed: true}, // Test 2 {policyData: `{ "Version":"2012-10-17", "Statement":[ { "Sid":"AddPerm", "Effect":"Allow", "Principal": "*", "Action":["s3:GetObject"], "Resource":["arn:aws:s3:::examplebucket/*"] } ] }`, shouldSucceed: true}, // Test 3 {policyData: `{ "Version": "2012-10-17", "Id": "S3PolicyId1", "Statement": [ { "Sid": "IPAllow", "Effect": "Allow", "Principal": "*", "Action": "s3:*", "Resource": "arn:aws:s3:::examplebucket/*", "Condition": { "IpAddress": {"aws:SourceIp": "54.240.143.0/24"}, "NotIpAddress": {"aws:SourceIp": "54.240.143.188/32"} } } ] }`, shouldSucceed: true}, // Test 4 {policyData: `{ "Id":"PolicyId2", "Version":"2012-10-17", "Statement":[ { "Sid":"AllowIPmix", "Effect":"Allow", "Principal":"*", "Action":"s3:*", "Resource":"arn:aws:s3:::examplebucket/*", "Condition": { "IpAddress": { "aws:SourceIp": [ "54.240.143.0/24", "2001:DB8:1234:5678::/64" ] }, "NotIpAddress": { "aws:SourceIp": [ "54.240.143.128/30", "2001:DB8:1234:5678:ABCD::/80" ] } } } ] }`, shouldSucceed: true}, // Test 5 {policyData: `{ "Version":"2012-10-17", "Id":"http referer policy example", "Statement":[ { "Sid":"Allow get requests originating from www.example.com and example.com.", "Effect":"Allow", "Principal":"*", "Action":"s3:GetObject", "Resource":"arn:aws:s3:::examplebucket/*", "Condition":{ "StringLike":{"aws:Referer":["http://www.example.com/*","http://example.com/*"]} } } ] }`, shouldSucceed: true}, // Test 6 {policyData: `{ "Version": "2012-10-17", "Id": "http referer policy example", "Statement": [ { "Sid": "Allow get requests referred by www.example.com and example.com.", "Effect": "Allow", "Principal": "*", "Action": "s3:GetObject", "Resource": "arn:aws:s3:::examplebucket/*", "Condition": { "StringLike": {"aws:Referer": ["http://www.example.com/*","http://example.com/*"]} } }, { "Sid": "Explicit deny to ensure requests are allowed only from specific referer.", "Effect": "Deny", "Principal": "*", "Action": "s3:*", "Resource": "arn:aws:s3:::examplebucket/*", "Condition": { "StringNotLike": {"aws:Referer": ["http://www.example.com/*","http://example.com/*"]} } } ] }`, shouldSucceed: true}, // Test 7 {policyData: `{ "Version":"2012-10-17", "Id":"PolicyForCloudFrontPrivateContent", "Statement":[ { "Sid":" Grant a CloudFront Origin Identity access to support private content", "Effect":"Allow", "Principal":{"CanonicalUser":"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be"}, "Action":"s3:GetObject", "Resource":"arn:aws:s3:::example-bucket/*" } ] }`, shouldSucceed: true}, // Test 8 {policyData: `{ "Version":"2012-10-17", "Statement":[ { "Sid":"111", "Effect":"Allow", "Principal":{"AWS":"1111111111"}, "Action":"s3:PutObject", "Resource":"arn:aws:s3:::examplebucket/*" }, { "Sid":"112", "Effect":"Deny", "Principal":{"AWS":"1111111111" }, "Action":"s3:PutObject", "Resource":"arn:aws:s3:::examplebucket/*", "Condition": { "StringNotEquals": {"s3:x-amz-grant-full-control":["emailAddress=xyz@amazon.com"]} } } ] }`, shouldSucceed: true}, // Test 9 {policyData: `{ "Version":"2012-10-17", "Statement":[ { "Sid":"InventoryAndAnalyticsExamplePolicy", "Effect":"Allow", "Principal": {"Service": "s3.amazonaws.com"}, "Action":["s3:PutObject"], "Resource":["arn:aws:s3:::destination-bucket/*"], "Condition": { "ArnLike": { "aws:SourceArn": "arn:aws:s3:::source-bucket" }, "StringEquals": { "aws:SourceAccount": "1234567890", "s3:x-amz-acl": "bucket-owner-full-control" } } } ] }`, shouldSucceed: true}, // Test 10 {policyData: `{ "Version": "2012-10-17", "Statement": [{ "Effect": "Deny", "Principal": { "AWS": [ "*" ] }, "Action": [ "s3:PutObject" ], "Resource": [ "arn:aws:s3:::mytest/*" ], "Condition": { "Null": { "s3:x-amz-server-side-encryption": [ true ] } } }] }`, shouldSucceed: true}, // Test 11 {policyData: `{ "Version": "2012-10-17", "Statement": [ { "Effect": "Deny", "Principal": "*", "Action": "s3:PutObject", "Resource": [ "arn:aws:s3:::DOC-EXAMPLE-BUCKET1", "arn:aws:s3:::DOC-EXAMPLE-BUCKET1/*" ], "Condition": { "NumericLessThan": { "s3:TlsVersion": 1.2 } } } ] }`, shouldSucceed: true}, } for i, testCase := range testCases { var policy BucketAccessPolicy err := json.Unmarshal([]byte(testCase.policyData), &policy) if testCase.shouldSucceed && err != nil { t.Fatalf("Test %d: expected to succeed but it has an error: %v", i+1, err) } if !testCase.shouldSucceed && err == nil { t.Fatalf("Test %d: expected to fail but succeeded", i+1) } } } // isValidStatement() is called and the result is validated. func TestIsValidStatement(t *testing.T) { testCases := []struct { statement Statement bucketName string expectedResult bool }{ // Empty statement and bucket name. {Statement{}, "", false}, // Empty statement. {Statement{}, "mybucket", false}, // Empty bucket name. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "", false}, // Statement with unknown actions. {Statement{ Actions: set.CreateStringSet("s3:ListBucketVersions"), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "mybucket", false}, // Statement with unknown effect. {Statement{ Actions: readOnlyBucketActions, Effect: "Deny", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "mybucket", false}, // Statement with nil Principal.AWS. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "mybucket", false}, // Statement with unknown Principal.AWS. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("arn:aws:iam::AccountNumberWithoutHyphens:root")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "mybucket", false}, // Statement with different bucket name. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), }, "mybucket", false}, // Statement with bucket name with suffixed string. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybuckettest/myobject"), }, "mybucket", false}, // Statement with bucket name and object name. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/myobject"), }, "mybucket", true}, // Statement with condition, bucket name and object name. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket/myobject"), }, "mybucket", true}, } for _, testCase := range testCases { if result := isValidStatement(testCase.statement, testCase.bucketName); result != testCase.expectedResult { t.Fatalf("%+v: expected: %t, got: %t", testCase, testCase.expectedResult, result) } } } // newStatements() is called and the result is validated. func TestNewStatements(t *testing.T) { testCases := []struct { policy BucketPolicy bucketName string prefix string expectedResult string }{ // BucketPolicyNone: with empty bucket name and prefix. {BucketPolicyNone, "", "", `[]`}, // BucketPolicyNone: with bucket name and empty prefix. {BucketPolicyNone, "mybucket", "", `[]`}, // BucketPolicyNone: with empty bucket name empty prefix. {BucketPolicyNone, "", "hello", `[]`}, // BucketPolicyNone: with bucket name prefix. {BucketPolicyNone, "mybucket", "hello", `[]`}, // BucketPolicyReadOnly: with empty bucket name and prefix. {BucketPolicyReadOnly, "", "", `[]`}, // BucketPolicyReadOnly: with bucket name and empty prefix. {BucketPolicyReadOnly, "mybucket", "", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, // BucketPolicyReadOnly: with empty bucket name empty prefix. {BucketPolicyReadOnly, "", "hello", `[]`}, // BucketPolicyReadOnly: with bucket name prefix. {BucketPolicyReadOnly, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Condition":{"StringLike":{"s3:prefix":["hello*"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, // BucketPolicyReadWrite: with empty bucket name and prefix. {BucketPolicyReadWrite, "", "", `[]`}, // BucketPolicyReadWrite: with bucket name and empty prefix. {BucketPolicyReadWrite, "mybucket", "", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, // BucketPolicyReadWrite: with empty bucket name empty prefix. {BucketPolicyReadWrite, "", "hello", `[]`}, // BucketPolicyReadWrite: with bucket name prefix. {BucketPolicyReadWrite, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Condition":{"StringLike":{"s3:prefix":["hello*"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, // BucketPolicyWriteOnly: with empty bucket name and prefix. {BucketPolicyWriteOnly, "", "", `[]`}, // BucketPolicyWriteOnly: with bucket name and empty prefix. {BucketPolicyWriteOnly, "mybucket", "", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, // BucketPolicyWriteOnly: with empty bucket name empty prefix. {BucketPolicyWriteOnly, "", "hello", `[]`}, // BucketPolicyWriteOnly: with bucket name prefix. {BucketPolicyWriteOnly, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, } for _, testCase := range testCases { statements := newStatements(testCase.policy, testCase.bucketName, testCase.prefix) if data, err := json.Marshal(statements); err == nil { if string(data) != testCase.expectedResult { t.Fatalf("%+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) } } } } // getInUsePolicy() is called and the result is validated. func TestGetInUsePolicy(t *testing.T) { testCases := []struct { statements []Statement bucketName string prefix string expectedResult1 bool expectedResult2 bool }{ // All empty statements, bucket name and prefix. {[]Statement{}, "", "", false, false}, // Non-empty statements, empty bucket name and empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "", "", false, false}, // Non-empty statements, non-empty bucket name and empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", false, false}, // Non-empty statements, empty bucket name and non-empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "", "hello", false, false}, // Empty statements, non-empty bucket name and empty prefix. {[]Statement{}, "mybucket", "", false, false}, // Empty statements, non-empty bucket name non-empty prefix. {[]Statement{}, "mybucket", "hello", false, false}, // Empty statements, empty bucket name and non-empty prefix. {[]Statement{}, "", "hello", false, false}, // Non-empty statements, non-empty bucket name, non-empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "hello", false, false}, // different bucket statements and empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), }}, "mybucket", "", false, false}, // different bucket statements. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), }}, "mybucket", "hello", false, false}, // different bucket multi-statements and empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), }, { Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::testbucket/world"), }}, "mybucket", "", false, false}, // different bucket multi-statements. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), }, { Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::testbucket/world"), }}, "mybucket", "hello", false, false}, // read-only in use. {[]Statement{{ Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "hello", true, false}, // write-only in use. {[]Statement{{ Actions: writeOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "hello", false, true}, // read-write in use. {[]Statement{{ Actions: readWriteObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "hello", true, true}, // read-write multi-statements. {[]Statement{{ Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }, { Actions: writeOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket/ground"), }}, "mybucket", "hello", true, true}, } for _, testCase := range testCases { result1, result2 := getInUsePolicy(testCase.statements, testCase.bucketName, testCase.prefix) if result1 != testCase.expectedResult1 || result2 != testCase.expectedResult2 { t.Fatalf("%+v: expected: [%t,%t], got: [%t,%t]", testCase, testCase.expectedResult1, testCase.expectedResult2, result1, result2) } } } // removeStatements() is called and the result is validated. func TestRemoveStatements(t *testing.T) { unknownCondMap1 := make(ConditionMap) unknownCondKeyMap1 := make(ConditionKeyMap) unknownCondKeyMap1.Add("s3:prefix", set.CreateStringSet("hello")) unknownCondMap1.Add("StringNotEquals", unknownCondKeyMap1) unknownCondMap11 := make(ConditionMap) unknownCondKeyMap11 := make(ConditionKeyMap) unknownCondKeyMap11.Add("s3:prefix", set.CreateStringSet("hello")) unknownCondMap11.Add("StringNotEquals", unknownCondKeyMap11) unknownCondMap12 := make(ConditionMap) unknownCondKeyMap12 := make(ConditionKeyMap) unknownCondKeyMap12.Add("s3:prefix", set.CreateStringSet("hello")) unknownCondMap12.Add("StringNotEquals", unknownCondKeyMap12) knownCondMap1 := make(ConditionMap) knownCondKeyMap1 := make(ConditionKeyMap) knownCondKeyMap1.Add("s3:prefix", set.CreateStringSet("hello")) knownCondMap1.Add("StringEquals", knownCondKeyMap1) knownCondMap11 := make(ConditionMap) knownCondKeyMap11 := make(ConditionKeyMap) knownCondKeyMap11.Add("s3:prefix", set.CreateStringSet("hello")) knownCondMap11.Add("StringEquals", knownCondKeyMap11) knownCondMap12 := make(ConditionMap) knownCondKeyMap12 := make(ConditionKeyMap) knownCondKeyMap12.Add("s3:prefix", set.CreateStringSet("hello")) knownCondMap12.Add("StringEquals", knownCondKeyMap12) knownCondMap13 := make(ConditionMap) knownCondKeyMap13 := make(ConditionKeyMap) knownCondKeyMap13.Add("s3:prefix", set.CreateStringSet("hello")) knownCondMap13.Add("StringEquals", knownCondKeyMap13) knownCondMap14 := make(ConditionMap) knownCondKeyMap14 := make(ConditionKeyMap) knownCondKeyMap14.Add("s3:prefix", set.CreateStringSet("hello")) knownCondMap14.Add("StringEquals", knownCondKeyMap14) knownCondMap2 := make(ConditionMap) knownCondKeyMap2 := make(ConditionKeyMap) knownCondKeyMap2.Add("s3:prefix", set.CreateStringSet("hello", "world")) knownCondMap2.Add("StringEquals", knownCondKeyMap2) testCases := []struct { statements []Statement bucketName string prefix string expectedResult string }{ // All empty statements, bucket name and prefix. {[]Statement{}, "", "", `[]`}, // Non-empty statements, empty bucket name and empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Non-empty statements, non-empty bucket name and empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Non-empty statements, empty bucket name and non-empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "", "hello", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Empty statements, non-empty bucket name and empty prefix. {[]Statement{}, "mybucket", "", `[]`}, // Empty statements, non-empty bucket name non-empty prefix. {[]Statement{}, "mybucket", "hello", `[]`}, // Empty statements, empty bucket name and non-empty prefix. {[]Statement{}, "", "hello", `[]`}, // Statement with unknown Actions with empty prefix. {[]Statement{{ Actions: set.CreateStringSet("s3:ListBucketVersions", "s3:ListAllMyBuckets"), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", `[{"Action":["s3:ListAllMyBuckets","s3:ListBucketVersions"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statement with unknown Actions. {[]Statement{{ Actions: set.CreateStringSet("s3:ListBucketVersions", "s3:ListAllMyBuckets"), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "hello", `[{"Action":["s3:ListAllMyBuckets","s3:ListBucketVersions"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statement with unknown Effect with empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Deny", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Effect":"Deny","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statement with unknown Effect. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Deny", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Effect":"Deny","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statement with unknown Principal.User.AWS with empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("arn:aws:iam::AccountNumberWithoutHyphens:root")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["arn:aws:iam::AccountNumberWithoutHyphens:root"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statement with unknown Principal.User.AWS. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("arn:aws:iam::AccountNumberWithoutHyphens:root")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["arn:aws:iam::AccountNumberWithoutHyphens:root"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statement with unknown Principal.User.CanonicalUser with empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{CanonicalUser: set.CreateStringSet("649262f44b8145cb")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"CanonicalUser":["649262f44b8145cb"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statement with unknown Principal.User.CanonicalUser. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{CanonicalUser: set.CreateStringSet("649262f44b8145cb")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"CanonicalUser":["649262f44b8145cb"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statement with unknown Conditions with empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statement with unknown Conditions. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statement with unknown Resource and empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::testbucket"],"Sid":""}]`}, // Statement with unknown Resource. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::testbucket"],"Sid":""}]`}, // Statement with known Actions with empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", `[]`}, // Statement with known Actions. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "hello", `[]`}, // Statement with known multiple Actions with empty prefix. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions).Union(commonBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", `[]`}, // Statement with known multiple Actions. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions).Union(commonBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "hello", `[]`}, // RemoveBucketActions with readOnlyInUse. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with prefix, readOnlyInUse. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with writeOnlyInUse. {[]Statement{{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: writeOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "", `[{"Action":["s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with prefix, writeOnlyInUse. {[]Statement{{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: writeOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with readOnlyInUse and writeOnlyInUse. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readWriteObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with prefix, readOnlyInUse and writeOnlyInUse. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readWriteObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with known Conditions, readOnlyInUse. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: knownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with prefix, known Conditions, readOnlyInUse. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: knownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "hello", `[{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with prefix, known Conditions contains other object prefix, readOnlyInUse. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: knownCondMap2, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Condition":{"StringEquals":{"s3:prefix":["world"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with unknown Conditions, readOnlyInUse. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with prefix, unknown Conditions, readOnlyInUse. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with known Conditions, writeOnlyInUse. {[]Statement{{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: knownCondMap11, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: writeOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with prefix, known Conditions, writeOnlyInUse. {[]Statement{{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: knownCondMap11, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: writeOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with unknown Conditions, writeOnlyInUse. {[]Statement{{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap11, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: writeOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with prefix, unknown Conditions, writeOnlyInUse. {[]Statement{{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap11, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: writeOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with known Conditions, readOnlyInUse and writeOnlyInUse. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: knownCondMap12, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with prefix, known Conditions, readOnlyInUse and writeOnlyInUse. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: knownCondMap12, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "hello", `[{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with unknown Conditions, readOnlyInUse and writeOnlyInUse. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap12, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // RemoveBucketActions with prefix, unknown Conditions, readOnlyInUse and writeOnlyInUse. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap12, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/world"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/world"],"Sid":""}]`}, // readOnlyObjectActions - RemoveObjectActions with known condition. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: knownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, // readOnlyObjectActions - RemoveObjectActions with prefix, known condition. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: knownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }}, "mybucket", "hello", `[]`}, // readOnlyObjectActions - RemoveObjectActions with unknown condition. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }}, "mybucket", "", `[{"Action":["s3:ListBucket"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, // readOnlyObjectActions - RemoveObjectActions with prefix, unknown condition. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucket"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // writeOnlyObjectActions - RemoveObjectActions with known condition. {[]Statement{{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: knownCondMap13, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: writeOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }}, "mybucket", "", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, // writeOnlyObjectActions - RemoveObjectActions with prefix, known condition. {[]Statement{{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: knownCondMap13, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: writeOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // writeOnlyObjectActions - RemoveObjectActions with unknown condition. {[]Statement{{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: writeOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }}, "mybucket", "", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, // writeOnlyObjectActions - RemoveObjectActions with prefix, unknown condition. {[]Statement{{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: writeOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // readWriteObjectActions - RemoveObjectActions with known condition. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: knownCondMap14, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }}, "mybucket", "", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, // readWriteObjectActions - RemoveObjectActions with prefix, known condition. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: knownCondMap13, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }}, "mybucket", "hello", `[]`}, // readWriteObjectActions - RemoveObjectActions with unknown condition. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }}, "mybucket", "", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, // readWriteObjectActions - RemoveObjectActions with prefix, unknown condition. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }}, "mybucket", "hello", `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringNotEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, } for _, testCase := range testCases { statements := removeStatements(testCase.statements, testCase.bucketName, testCase.prefix) if data, err := json.Marshal(statements); err != nil { t.Fatalf("unable encoding to json, %s", err) } else if string(data) != testCase.expectedResult { t.Fatalf("%+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) } } } // appendStatement() is called and the result is validated. func TestAppendStatement(t *testing.T) { condMap := make(ConditionMap) condKeyMap := make(ConditionKeyMap) condKeyMap.Add("s3:prefix", set.CreateStringSet("hello")) condMap.Add("StringEquals", condKeyMap) condMap1 := make(ConditionMap) condKeyMap1 := make(ConditionKeyMap) condKeyMap1.Add("s3:prefix", set.CreateStringSet("world")) condMap1.Add("StringEquals", condKeyMap1) unknownCondMap1 := make(ConditionMap) unknownCondKeyMap1 := make(ConditionKeyMap) unknownCondKeyMap1.Add("s3:prefix", set.CreateStringSet("world")) unknownCondMap1.Add("StringNotEquals", unknownCondKeyMap1) testCases := []struct { statements []Statement statement Statement expectedResult string }{ // Empty statements and empty new statement. {[]Statement{}, Statement{}, `[]`}, // Non-empty statements and empty new statement. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, Statement{}, `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Empty statements and non-empty new statement. {[]Statement{}, Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Append existing statement. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Append same statement with different resource. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), }, `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket","arn:aws:s3:::testbucket"],"Sid":""}]`}, // Append same statement with different actions. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, Statement{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Elements of new statement contains elements in statements. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket", "arn:aws:s3:::testbucket"), }}, Statement{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket","arn:aws:s3:::testbucket"],"Sid":""}]`}, // Elements of new statement with conditions contains elements in statements. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: condMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket", "arn:aws:s3:::testbucket"), }}, Statement{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: condMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket","arn:aws:s3:::testbucket"],"Sid":""}]`}, // Statements with condition and new statement with condition. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: condMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket", "arn:aws:s3:::testbucket"), }}, Statement{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: condMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket","arn:aws:s3:::testbucket"],"Sid":""},{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["world"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statements with condition and same resources, and new statement with condition. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: condMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, Statement{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: condMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello","world"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statements with unknown condition and same resources, and new statement with known condition. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: unknownCondMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, Statement{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: condMap1, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["world"]},"StringNotEquals":{"s3:prefix":["world"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statements without condition and new statement with condition. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket", "arn:aws:s3:::testbucket"), }}, Statement{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: condMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket","arn:aws:s3:::testbucket"],"Sid":""},{"Action":["s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statements with condition and new statement without condition. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: condMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket", "arn:aws:s3:::testbucket"), }}, Statement{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, `[{"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Condition":{"StringEquals":{"s3:prefix":["hello"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket","arn:aws:s3:::testbucket"],"Sid":""},{"Action":["s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // Statements and new statement are different. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, Statement{ Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }, `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, } for _, testCase := range testCases { statements := appendStatement(testCase.statements, testCase.statement) if data, err := json.Marshal(statements); err != nil { t.Fatalf("unable encoding to json, %s", err) } else if string(data) != testCase.expectedResult { t.Fatalf("%+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) } } } // getBucketPolicy() is called and the result is validated. func TestGetBucketPolicy(t *testing.T) { helloCondMap := make(ConditionMap) helloCondKeyMap := make(ConditionKeyMap) helloCondKeyMap.Add("s3:prefix", set.CreateStringSet("hello")) helloCondMap.Add("StringEquals", helloCondKeyMap) worldCondMap := make(ConditionMap) worldCondKeyMap := make(ConditionKeyMap) worldCondKeyMap.Add("s3:prefix", set.CreateStringSet("world")) worldCondMap.Add("StringEquals", worldCondKeyMap) notHelloCondMap := make(ConditionMap) notHelloCondMap.Add("StringNotEquals", worldCondKeyMap) // StringLike condition map for "hello*" stringLikeHelloCondMap := make(ConditionMap) stringLikeHelloCondKeyMap := make(ConditionKeyMap) stringLikeHelloCondKeyMap.Add("s3:prefix", set.CreateStringSet("hello*")) stringLikeHelloCondMap.Add("StringLike", stringLikeHelloCondKeyMap) // StringLike condition map for "world*" stringLikeWorldCondMap := make(ConditionMap) stringLikeWorldCondKeyMap := make(ConditionKeyMap) stringLikeWorldCondKeyMap.Add("s3:prefix", set.CreateStringSet("world*")) stringLikeWorldCondMap.Add("StringLike", stringLikeWorldCondKeyMap) // StringNotLike condition map for "hello*" stringNotLikeHelloCondMap := make(ConditionMap) stringNotLikeHelloCondKeyMap := make(ConditionKeyMap) stringNotLikeHelloCondKeyMap.Add("s3:prefix", set.CreateStringSet("hello*")) stringNotLikeHelloCondMap.Add("StringNotLike", stringNotLikeHelloCondKeyMap) // StringNotLike condition map for "world*" stringNotLikeWorldCondMap := make(ConditionMap) stringNotLikeWorldCondKeyMap := make(ConditionKeyMap) stringNotLikeWorldCondKeyMap.Add("s3:prefix", set.CreateStringSet("world*")) stringNotLikeWorldCondMap.Add("StringNotLike", stringNotLikeWorldCondKeyMap) testCases := []struct { statement Statement prefix string expectedResult1 bool expectedResult2 bool expectedResult3 bool }{ // Statement with invalid Effect. {Statement{ Actions: readOnlyBucketActions, Effect: "Deny", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "", false, false, false}, // Statement with invalid Effect with prefix. {Statement{ Actions: readOnlyBucketActions, Effect: "Deny", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", false, false, false}, // Statement with invalid Principal.AWS. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("arn:aws:iam::AccountNumberWithoutHyphens:root")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "", false, false, false}, // Statement with invalid Principal.AWS with prefix. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("arn:aws:iam::AccountNumberWithoutHyphens:root")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", false, false, false}, // Statement with commonBucketActions. {Statement{ Actions: commonBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "", true, false, false}, // Statement with commonBucketActions. {Statement{ Actions: commonBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", true, false, false}, // Statement with commonBucketActions and condition. {Statement{ Actions: commonBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "", false, false, false}, // Statement with commonBucketActions and condition. {Statement{ Actions: commonBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", false, false, false}, // Statement with writeOnlyBucketActions. {Statement{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "", false, false, true}, // Statement with writeOnlyBucketActions. {Statement{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", false, false, true}, // Statement with writeOnlyBucketActions and condition {Statement{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "", false, false, false}, // Statement with writeOnlyBucketActions and condition. {Statement{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", false, false, false}, // Statement with readOnlyBucketActions. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "", false, true, false}, // Statement with readOnlyBucketActions. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", false, true, false}, // Statement with readOnlyBucketActions with empty condition. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "", false, false, false}, // Statement with readOnlyBucketActions with empty condition. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", false, false, false}, // Statement with readOnlyBucketActions with matching condition. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: helloCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "", false, false, false}, // Statement with readOnlyBucketActions with matching condition. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: helloCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", false, true, false}, // Statement with readOnlyBucketActions with different condition. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: worldCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "", false, false, false}, // Statement with readOnlyBucketActions with different condition. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: worldCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", false, false, false}, // Statement with readOnlyBucketActions with StringNotEquals condition. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: notHelloCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "", false, false, false}, // Statement with readOnlyBucketActions with StringNotEquals condition. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: notHelloCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", false, true, false}, // Statement with StringLike condition for "hello*" pattern with empty prefix - should not grant readOnly access. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: stringLikeHelloCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "", false, false, false}, // Statement with StringLike condition for "hello*" pattern with "hello" prefix - should grant readOnly access. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: stringLikeHelloCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", false, true, false}, // Statement with StringLike condition for "world*" pattern with "hello" prefix - should not grant readOnly access. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: stringLikeWorldCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", false, false, false}, // Statement with StringNotLike condition for "hello*" pattern with empty prefix - should not grant readOnly access. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: stringNotLikeHelloCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "", false, false, false}, // Statement with StringNotLike condition for "hello*" pattern with "hello" prefix - prefix matches pattern so should not grant readOnly access. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: stringNotLikeHelloCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", false, false, false}, // Statement with StringNotLike condition for "world*" pattern with "hello" prefix - prefix doesn't match pattern so should grant readOnly access. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: stringNotLikeWorldCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "hello", false, true, false}, // Statement with StringNotLike condition for "world*" pattern with "world" prefix - prefix matches pattern so should not grant readOnly access. {Statement{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: stringNotLikeWorldCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, "world", false, false, false}, } for _, testCase := range testCases { commonFound, readOnly, writeOnly := getBucketPolicy(testCase.statement, testCase.prefix) if testCase.expectedResult1 != commonFound || testCase.expectedResult2 != readOnly || testCase.expectedResult3 != writeOnly { t.Fatalf("%+v: expected: [%t,%t,%t], got: [%t,%t,%t]", testCase, testCase.expectedResult1, testCase.expectedResult2, testCase.expectedResult3, commonFound, readOnly, writeOnly) } } } // getObjectPolicy() is called and the result is validated. func TestGetObjectPolicy(t *testing.T) { testCases := []struct { statement Statement expectedResult1 bool expectedResult2 bool }{ // Statement with invalid Effect. {Statement{ Actions: readOnlyObjectActions, Effect: "Deny", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }, false, false}, // Statement with invalid Principal.AWS. {Statement{ Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("arn:aws:iam::AccountNumberWithoutHyphens:root")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }, false, false}, // Statement with condition. {Statement{ Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: make(ConditionMap), Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }, false, false}, // Statement with readOnlyObjectActions. {Statement{ Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }, true, false}, // Statement with writeOnlyObjectActions. {Statement{ Actions: writeOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }, false, true}, // Statement with readOnlyObjectActions and writeOnlyObjectActions. {Statement{ Actions: readOnlyObjectActions.Union(writeOnlyObjectActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/hello*"), }, true, true}, } for _, testCase := range testCases { readOnly, writeOnly := getObjectPolicy(testCase.statement) if testCase.expectedResult1 != readOnly || testCase.expectedResult2 != writeOnly { t.Fatalf("%+v: expected: [%t,%t], got: [%t,%t]", testCase, testCase.expectedResult1, testCase.expectedResult2, readOnly, writeOnly) } } } // GetPolicyRules is called and the result is validated func TestListBucketPolicies(t *testing.T) { // Condition for read objects downloadCondMap := make(ConditionMap) downloadCondKeyMap := make(ConditionKeyMap) downloadCondKeyMap.Add("s3:prefix", set.CreateStringSet("download")) downloadCondMap.Add("StringEquals", downloadCondKeyMap) // Condition for readwrite objects downloadUploadCondMap := make(ConditionMap) downloadUploadCondKeyMap := make(ConditionKeyMap) downloadUploadCondKeyMap.Add("s3:prefix", set.CreateStringSet("both")) downloadUploadCondMap.Add("StringEquals", downloadUploadCondKeyMap) commonSetActions := commonBucketActions.Union(readOnlyBucketActions) testCases := []struct { statements []Statement bucketName string prefix string expectedResult map[string]BucketPolicy }{ // Empty statements, bucket name and prefix. {[]Statement{}, "", "", map[string]BucketPolicy{}}, // Non-empty statements, empty bucket name and empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "", "", map[string]BucketPolicy{}}, // Empty statements, non-empty bucket name and empty prefix. {[]Statement{}, "mybucket", "", map[string]BucketPolicy{}}, // Readonly object statement {[]Statement{ { Actions: commonBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: downloadCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/download*"), }, }, "mybucket", "", map[string]BucketPolicy{"mybucket/download*": BucketPolicyReadOnly}}, {[]Statement{ { Actions: commonSetActions.Union(readOnlyObjectActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket", "arn:aws:s3:::mybucket/*"), }, }, "mybucket", "", map[string]BucketPolicy{"mybucket/*": BucketPolicyReadOnly}}, // Write Only {[]Statement{ { Actions: commonBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: writeOnlyObjectActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/upload*"), }, }, "mybucket", "", map[string]BucketPolicy{"mybucket/upload*": BucketPolicyWriteOnly}}, // Readwrite {[]Statement{ { Actions: commonBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: downloadUploadCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }, { Actions: writeOnlyObjectActions.Union(readOnlyObjectActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket/both*"), }, }, "mybucket", "", map[string]BucketPolicy{"mybucket/both*": BucketPolicyReadWrite}}, } for _, testCase := range testCases { policyRules := GetPolicies(testCase.statements, testCase.bucketName, "") if !reflect.DeepEqual(testCase.expectedResult, policyRules) { t.Fatalf("%+v:\n expected: %+v, got: %+v", testCase, testCase.expectedResult, policyRules) } } } // GetPolicy() is called and the result is validated. func TestGetPolicy(t *testing.T) { helloCondMap := make(ConditionMap) helloCondKeyMap := make(ConditionKeyMap) helloCondKeyMap.Add("s3:prefix", set.CreateStringSet("hello")) helloCondMap.Add("StringEquals", helloCondKeyMap) testCases := []struct { statements []Statement bucketName string prefix string expectedResult BucketPolicy }{ // Empty statements, bucket name and prefix. {[]Statement{}, "", "", BucketPolicyNone}, // Non-empty statements, empty bucket name and empty prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "", "", BucketPolicyNone}, // Empty statements, non-empty bucket name and empty prefix. {[]Statement{}, "mybucket", "", BucketPolicyNone}, // not-matching Statements. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), }}, "mybucket", "", BucketPolicyNone}, // not-matching Statements with prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), }}, "mybucket", "hello", BucketPolicyNone}, // Statements with only commonBucketActions. {[]Statement{{ Actions: commonBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", BucketPolicyNone}, // Statements with only commonBucketActions with prefix. {[]Statement{{ Actions: commonBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "hello", BucketPolicyNone}, // Statements with only readOnlyBucketActions. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", BucketPolicyNone}, // Statements with only readOnlyBucketActions with prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "hello", BucketPolicyNone}, // Statements with only readOnlyBucketActions with conditions. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: helloCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", BucketPolicyNone}, // Statements with only readOnlyBucketActions with prefix with conditons. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: helloCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "hello", BucketPolicyNone}, // Statements with only writeOnlyBucketActions. {[]Statement{{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", BucketPolicyNone}, // Statements with only writeOnlyBucketActions with prefix. {[]Statement{{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "hello", BucketPolicyNone}, // Statements with only readOnlyBucketActions + writeOnlyBucketActions. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", BucketPolicyNone}, // Statements with only readOnlyBucketActions + writeOnlyBucketActions with prefix. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "hello", BucketPolicyNone}, // Statements with only readOnlyBucketActions + writeOnlyBucketActions and conditions. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: helloCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "", BucketPolicyNone}, // Statements with only readOnlyBucketActions + writeOnlyBucketActions and conditions with prefix. {[]Statement{{ Actions: readOnlyBucketActions.Union(writeOnlyBucketActions), Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: helloCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, "mybucket", "hello", BucketPolicyNone}, } for _, testCase := range testCases { policy := GetPolicy(testCase.statements, testCase.bucketName, testCase.prefix) if testCase.expectedResult != policy { t.Fatalf("%+v: expected: %s, got: %s", testCase, testCase.expectedResult, policy) } } } // SetPolicy() is called and the result is validated. func TestSetPolicy(t *testing.T) { helloCondMap := make(ConditionMap) helloCondKeyMap := make(ConditionKeyMap) helloCondKeyMap.Add("s3:prefix", set.CreateStringSet("hello")) helloCondMap.Add("StringEquals", helloCondKeyMap) testCases := []struct { statements []Statement policy BucketPolicy bucketName string prefix string expectedResult string }{ // BucketPolicyNone - empty statements, bucket name and prefix. {[]Statement{}, BucketPolicyNone, "", "", `[]`}, // BucketPolicyNone - non-empty statements, bucket name and prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, BucketPolicyNone, "", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""}]`}, // BucketPolicyNone - empty statements, non-empty bucket name and prefix. {[]Statement{}, BucketPolicyNone, "mybucket", "", `[]`}, // BucketPolicyNone - empty statements, bucket name and non-empty prefix. {[]Statement{}, BucketPolicyNone, "", "hello", `[]`}, // BucketPolicyReadOnly - empty statements, bucket name and prefix. {[]Statement{}, BucketPolicyReadOnly, "", "", `[]`}, // BucketPolicyReadOnly - non-empty statements, bucket name and prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), }}, BucketPolicyReadOnly, "", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::testbucket"],"Sid":""}]`}, // BucketPolicyReadOnly - empty statements, non-empty bucket name and prefix. {[]Statement{}, BucketPolicyReadOnly, "mybucket", "", `[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, // BucketPolicyReadOnly - empty statements, bucket name and non-empty prefix. {[]Statement{}, BucketPolicyReadOnly, "", "hello", `[]`}, // BucketPolicyReadOnly - empty statements, non-empty bucket name and non-empty prefix. {[]Statement{}, BucketPolicyReadOnly, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Condition":{"StringLike":{"s3:prefix":["hello*"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, // BucketPolicyWriteOnly - empty statements, bucket name and prefix. {[]Statement{}, BucketPolicyReadOnly, "", "", `[]`}, // BucketPolicyWriteOnly - non-empty statements, bucket name and prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), }}, BucketPolicyWriteOnly, "", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::testbucket"],"Sid":""}]`}, // BucketPolicyWriteOnly - empty statements, non-empty bucket name and prefix. {[]Statement{}, BucketPolicyWriteOnly, "mybucket", "", `[{"Action":["s3:GetBucketLocation","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, // BucketPolicyWriteOnly - empty statements, bucket name and non-empty prefix. {[]Statement{}, BucketPolicyWriteOnly, "", "hello", `[]`}, // BucketPolicyWriteOnly - empty statements, non-empty bucket name and non-empty prefix. {[]Statement{}, BucketPolicyWriteOnly, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, // BucketPolicyReadWrite - empty statements, bucket name and prefix. {[]Statement{}, BucketPolicyReadWrite, "", "", `[]`}, // BucketPolicyReadWrite - non-empty statements, bucket name and prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::testbucket"), }}, BucketPolicyReadWrite, "", "", `[{"Action":["s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::testbucket"],"Sid":""}]`}, // BucketPolicyReadWrite - empty statements, non-empty bucket name and prefix. {[]Statement{}, BucketPolicyReadWrite, "mybucket", "", `[{"Action":["s3:GetBucketLocation","s3:ListBucket","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, // BucketPolicyReadWrite - empty statements, bucket name and non-empty prefix. {[]Statement{}, BucketPolicyReadWrite, "", "hello", `[]`}, // BucketPolicyReadWrite - empty statements, non-empty bucket name and non-empty prefix. {[]Statement{}, BucketPolicyReadWrite, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Condition":{"StringLike":{"s3:prefix":["hello*"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, // Set readonly. {[]Statement{{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, BucketPolicyReadOnly, "mybucket", "", `[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, // Set readonly with prefix. {[]Statement{{ Actions: writeOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, BucketPolicyReadOnly, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Condition":{"StringLike":{"s3:prefix":["hello*"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, // Set writeonly. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, BucketPolicyWriteOnly, "mybucket", "", `[{"Action":["s3:GetBucketLocation","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, // Set writeonly with prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: helloCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, BucketPolicyWriteOnly, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, // Set readwrite. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, BucketPolicyReadWrite, "mybucket", "", `[{"Action":["s3:GetBucketLocation","s3:ListBucket","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/*"],"Sid":""}]`}, // Set readwrite with prefix. {[]Statement{{ Actions: readOnlyBucketActions, Effect: "Allow", Principal: User{AWS: set.CreateStringSet("*")}, Conditions: helloCondMap, Resources: set.CreateStringSet("arn:aws:s3:::mybucket"), }}, BucketPolicyReadWrite, "mybucket", "hello", `[{"Action":["s3:GetBucketLocation","s3:ListBucketMultipartUploads"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:ListBucket"],"Condition":{"StringLike":{"s3:prefix":["hello*"]}},"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket"],"Sid":""},{"Action":["s3:AbortMultipartUpload","s3:DeleteObject","s3:GetObject","s3:ListMultipartUploadParts","s3:PutObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::mybucket/hello*"],"Sid":""}]`}, } for _, testCase := range testCases { statements := SetPolicy(testCase.statements, testCase.policy, testCase.bucketName, testCase.prefix) if data, err := json.Marshal(statements); err != nil { t.Fatalf("unable encoding to json, %s", err) } else if string(data) != testCase.expectedResult { t.Fatalf("%+v: expected: %s, got: %s", testCase, testCase.expectedResult, string(data)) } } } // Validates bucket policy string. func TestIsValidBucketPolicy(t *testing.T) { testCases := []struct { inputPolicy BucketPolicy expectedResult bool }{ // valid inputs. {BucketPolicy("none"), true}, {BucketPolicy("readonly"), true}, {BucketPolicy("readwrite"), true}, {BucketPolicy("writeonly"), true}, // invalid input. {BucketPolicy("readwriteonly"), false}, {BucketPolicy("writeread"), false}, } for i, testCase := range testCases { actualResult := testCase.inputPolicy.IsValidBucketPolicy() if testCase.expectedResult != actualResult { t.Errorf("Test %d: Expected IsValidBucket policy to be '%v' for policy \"%s\", but instead found it to be '%v'", i+1, testCase.expectedResult, testCase.inputPolicy, actualResult) } } } // Tests validate Bucket policy resource matcher. func TestBucketPolicyResourceMatch(t *testing.T) { // generates\ statement with given resource.. generateStatement := func(resource string) Statement { statement := Statement{} statement.Resources = set.CreateStringSet(resource) return statement } // generates resource prefix. generateResource := func(bucketName, objectName string) string { return awsResourcePrefix + bucketName + "/" + objectName } testCases := []struct { resourceToMatch string statement Statement expectedResourceMatch bool }{ // Test case 1-4. // Policy with resource ending with bucket/* allows access to all objects inside the given bucket. {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/*")), true}, {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/*")), true}, {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/*")), true}, {generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/*")), true}, // Test case - 5. // Policy with resource ending with bucket/oo* should not allow access to bucket/output.txt. {generateResource("minio-bucket", "output.txt"), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/oo*")), false}, // Test case - 6. // Policy with resource ending with bucket/oo* should allow access to bucket/ootput.txt. {generateResource("minio-bucket", "ootput.txt"), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/oo*")), true}, // Test case - 7. // Policy with resource ending with bucket/oo* allows access to all subfolders starting with "oo" inside given bucket. {generateResource("minio-bucket", "oop-bucket/my-file"), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/oo*")), true}, // Test case - 8. {generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/Asia/Japan/*")), false}, // Test case - 9. {generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/Asia/Japan/*")), false}, // Test case - 10. // Proves that the name space is flat. {generateResource("minio-bucket", "Africa/Bihar/India/design_info.doc/Bihar"), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/*/India/*/Bihar")), true}, // Test case - 11. // Proves that the name space is flat. {generateResource("minio-bucket", "Asia/China/India/States/Bihar/output.txt"), generateStatement(fmt.Sprintf("%s%s", awsResourcePrefix, "minio-bucket"+"/*/India/*/Bihar/*")), true}, } for i, testCase := range testCases { resources := testCase.statement.Resources.FuncMatch(resourceMatch, testCase.resourceToMatch) actualResourceMatch := resources.Equals(testCase.statement.Resources) if testCase.expectedResourceMatch != actualResourceMatch { t.Errorf("Test %d: Expected Resource match to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResourceMatch, actualResourceMatch) } } } minio-go-7.0.97/pkg/replication/000077500000000000000000000000001510244170000164205ustar00rootroot00000000000000minio-go-7.0.97/pkg/replication/replication.go000066400000000000000000000752371510244170000212760ustar00rootroot00000000000000/* * MinIO Client (C) 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package replication import ( "bytes" "encoding/xml" "fmt" "math" "strconv" "strings" "time" "unicode/utf8" "github.com/rs/xid" ) var errInvalidFilter = fmt.Errorf("invalid filter") // OptionType specifies operation to be performed on config type OptionType string const ( // AddOption specifies addition of rule to config AddOption OptionType = "Add" // SetOption specifies modification of existing rule to config SetOption OptionType = "Set" // RemoveOption specifies rule options are for removing a rule RemoveOption OptionType = "Remove" // ImportOption is for getting current config ImportOption OptionType = "Import" ) // Options represents options to set a replication configuration rule type Options struct { Op OptionType RoleArn string ID string Prefix string RuleStatus string Priority string TagString string StorageClass string DestBucket string IsTagSet bool IsSCSet bool ReplicateDeletes string // replicate versioned deletes ReplicateDeleteMarkers string // replicate soft deletes ReplicaSync string // replicate replica metadata modifications ExistingObjectReplicate string } // Tags returns a slice of tags for a rule func (opts Options) Tags() ([]Tag, error) { var tagList []Tag tagTokens := strings.Split(opts.TagString, "&") for _, tok := range tagTokens { if tok == "" { break } kv := strings.SplitN(tok, "=", 2) if len(kv) != 2 { return []Tag{}, fmt.Errorf("tags should be entered as comma separated k=v pairs") } tagList = append(tagList, Tag{ Key: kv[0], Value: kv[1], }) } return tagList, nil } // Config - replication configuration specified in // https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html type Config struct { XMLName xml.Name `xml:"ReplicationConfiguration" json:"-"` Rules []Rule `xml:"Rule" json:"Rules"` Role string `xml:"Role" json:"Role"` } // Empty returns true if config is not set func (c *Config) Empty() bool { return len(c.Rules) == 0 } // AddRule adds a new rule to existing replication config. If a rule exists with the // same ID, then the rule is replaced. func (c *Config) AddRule(opts Options) error { priority, err := strconv.Atoi(opts.Priority) if err != nil { return err } var compatSw bool // true if RoleArn is used with new mc client and older minio version prior to multisite if opts.RoleArn != "" { tokens := strings.Split(opts.RoleArn, ":") if len(tokens) != 6 { return fmt.Errorf("invalid format for replication Role Arn: %v", opts.RoleArn) } switch { case strings.HasPrefix(opts.RoleArn, "arn:minio:replication") && len(c.Rules) == 0: c.Role = opts.RoleArn compatSw = true case strings.HasPrefix(opts.RoleArn, "arn:aws:iam"): c.Role = opts.RoleArn default: return fmt.Errorf("RoleArn invalid for AWS replication configuration: %v", opts.RoleArn) } } var status Status // toggle rule status for edit option switch opts.RuleStatus { case "enable": status = Enabled case "disable": status = Disabled default: return fmt.Errorf("rule state should be either [enable|disable]") } tags, err := opts.Tags() if err != nil { return err } andVal := And{ Tags: tags, } filter := Filter{Prefix: opts.Prefix} // only a single tag is set. if opts.Prefix == "" && len(tags) == 1 { filter.Tag = tags[0] } // both prefix and tag are present if len(andVal.Tags) > 1 || opts.Prefix != "" { filter.And = andVal filter.And.Prefix = opts.Prefix filter.Prefix = "" filter.Tag = Tag{} } if opts.ID == "" { opts.ID = xid.New().String() } destBucket := opts.DestBucket // ref https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html if btokens := strings.Split(destBucket, ":"); len(btokens) != 6 { if len(btokens) == 1 && compatSw { destBucket = fmt.Sprintf("arn:aws:s3:::%s", destBucket) } else { return fmt.Errorf("destination bucket needs to be in Arn format") } } dmStatus := Disabled if opts.ReplicateDeleteMarkers != "" { switch opts.ReplicateDeleteMarkers { case "enable": dmStatus = Enabled case "disable": dmStatus = Disabled default: return fmt.Errorf("ReplicateDeleteMarkers should be either enable|disable") } } vDeleteStatus := Disabled if opts.ReplicateDeletes != "" { switch opts.ReplicateDeletes { case "enable": vDeleteStatus = Enabled case "disable": vDeleteStatus = Disabled default: return fmt.Errorf("ReplicateDeletes should be either enable|disable") } } var replicaSync Status // replica sync is by default Enabled, unless specified. switch opts.ReplicaSync { case "enable", "": replicaSync = Enabled case "disable": replicaSync = Disabled default: return fmt.Errorf("replica metadata sync should be either [enable|disable]") } var existingStatus Status if opts.ExistingObjectReplicate != "" { switch opts.ExistingObjectReplicate { case "enable": existingStatus = Enabled case "disable", "": existingStatus = Disabled default: return fmt.Errorf("existingObjectReplicate should be either enable|disable") } } newRule := Rule{ ID: opts.ID, Priority: priority, Status: status, Filter: filter, Destination: Destination{ Bucket: destBucket, StorageClass: opts.StorageClass, }, DeleteMarkerReplication: DeleteMarkerReplication{Status: dmStatus}, DeleteReplication: DeleteReplication{Status: vDeleteStatus}, // MinIO enables replica metadata syncing by default in the case of bi-directional replication to allow // automatic failover as the expectation in this case is that replica and source should be identical. // However AWS leaves this configurable https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-for-metadata-changes.html SourceSelectionCriteria: SourceSelectionCriteria{ ReplicaModifications: ReplicaModifications{ Status: replicaSync, }, }, // By default disable existing object replication unless selected ExistingObjectReplication: ExistingObjectReplication{ Status: existingStatus, }, } // validate rule after overlaying priority for pre-existing rule being disabled. if err := newRule.Validate(); err != nil { return err } // if replication config uses RoleArn, migrate this to the destination element as target ARN for remote bucket for MinIO configuration if c.Role != "" && !strings.HasPrefix(c.Role, "arn:aws:iam") && !compatSw { for i := range c.Rules { c.Rules[i].Destination.Bucket = c.Role } c.Role = "" } for _, rule := range c.Rules { if rule.Priority == newRule.Priority { return fmt.Errorf("priority must be unique. Replication configuration already has a rule with this priority") } if rule.ID == newRule.ID { return fmt.Errorf("a rule exists with this ID") } } c.Rules = append(c.Rules, newRule) return nil } // EditRule modifies an existing rule in replication config func (c *Config) EditRule(opts Options) error { if opts.ID == "" { return fmt.Errorf("rule ID missing") } // if replication config uses RoleArn, migrate this to the destination element as target ARN for remote bucket for non AWS. if c.Role != "" && !strings.HasPrefix(c.Role, "arn:aws:iam") && len(c.Rules) > 1 { for i := range c.Rules { c.Rules[i].Destination.Bucket = c.Role } c.Role = "" } rIdx := -1 var newRule Rule for i, rule := range c.Rules { if rule.ID == opts.ID { rIdx = i newRule = rule break } } if rIdx < 0 { return fmt.Errorf("rule with ID %s not found in replication configuration", opts.ID) } prefixChg := opts.Prefix != newRule.Prefix() if opts.IsTagSet || prefixChg { prefix := newRule.Prefix() if prefix != opts.Prefix { prefix = opts.Prefix } tags := []Tag{newRule.Filter.Tag} if len(newRule.Filter.And.Tags) != 0 { tags = newRule.Filter.And.Tags } var err error if opts.IsTagSet { tags, err = opts.Tags() if err != nil { return err } } andVal := And{ Tags: tags, } filter := Filter{Prefix: prefix} // only a single tag is set. if prefix == "" && len(tags) == 1 { filter.Tag = tags[0] } // both prefix and tag are present if len(andVal.Tags) > 1 || prefix != "" { filter.And = andVal filter.And.Prefix = prefix filter.Prefix = "" filter.Tag = Tag{} } newRule.Filter = filter } // toggle rule status for edit option if opts.RuleStatus != "" { switch opts.RuleStatus { case "enable": newRule.Status = Enabled case "disable": newRule.Status = Disabled default: return fmt.Errorf("rule state should be either [enable|disable]") } } // set DeleteMarkerReplication rule status for edit option if opts.ReplicateDeleteMarkers != "" { switch opts.ReplicateDeleteMarkers { case "enable": newRule.DeleteMarkerReplication.Status = Enabled case "disable": newRule.DeleteMarkerReplication.Status = Disabled default: return fmt.Errorf("ReplicateDeleteMarkers state should be either [enable|disable]") } } // set DeleteReplication rule status for edit option. This is a MinIO specific // option to replicate versioned deletes if opts.ReplicateDeletes != "" { switch opts.ReplicateDeletes { case "enable": newRule.DeleteReplication.Status = Enabled case "disable": newRule.DeleteReplication.Status = Disabled default: return fmt.Errorf("ReplicateDeletes state should be either [enable|disable]") } } if opts.ReplicaSync != "" { switch opts.ReplicaSync { case "enable", "": newRule.SourceSelectionCriteria.ReplicaModifications.Status = Enabled case "disable": newRule.SourceSelectionCriteria.ReplicaModifications.Status = Disabled default: return fmt.Errorf("replica metadata sync should be either [enable|disable]") } } if opts.ExistingObjectReplicate != "" { switch opts.ExistingObjectReplicate { case "enable": newRule.ExistingObjectReplication.Status = Enabled case "disable": newRule.ExistingObjectReplication.Status = Disabled default: return fmt.Errorf("existingObjectsReplication state should be either [enable|disable]") } } if opts.IsSCSet { newRule.Destination.StorageClass = opts.StorageClass } if opts.Priority != "" { priority, err := strconv.Atoi(opts.Priority) if err != nil { return err } newRule.Priority = priority } if opts.DestBucket != "" { destBucket := opts.DestBucket // ref https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html if btokens := strings.Split(opts.DestBucket, ":"); len(btokens) != 6 { return fmt.Errorf("destination bucket needs to be in Arn format") } newRule.Destination.Bucket = destBucket } // validate rule if err := newRule.Validate(); err != nil { return err } // ensure priority and destination bucket restrictions are not violated for idx, rule := range c.Rules { if rule.Priority == newRule.Priority && rIdx != idx { return fmt.Errorf("priority must be unique. Replication configuration already has a rule with this priority") } if rule.Destination.Bucket != newRule.Destination.Bucket && rule.ID == newRule.ID { if c.Role == newRule.Destination.Bucket { continue } return fmt.Errorf("invalid destination bucket for this rule") } } c.Rules[rIdx] = newRule return nil } // RemoveRule removes a rule from replication config. func (c *Config) RemoveRule(opts Options) error { var newRules []Rule ruleFound := false for _, rule := range c.Rules { if rule.ID != opts.ID { newRules = append(newRules, rule) continue } ruleFound = true } if !ruleFound { return fmt.Errorf("Rule with ID %s not found", opts.ID) } if len(newRules) == 0 { return fmt.Errorf("replication configuration should have at least one rule") } c.Rules = newRules return nil } // Rule - a rule for replication configuration. type Rule struct { XMLName xml.Name `xml:"Rule" json:"-"` ID string `xml:"ID,omitempty"` Status Status `xml:"Status"` Priority int `xml:"Priority"` DeleteMarkerReplication DeleteMarkerReplication `xml:"DeleteMarkerReplication"` DeleteReplication DeleteReplication `xml:"DeleteReplication"` Destination Destination `xml:"Destination"` Filter Filter `xml:"Filter" json:"Filter"` SourceSelectionCriteria SourceSelectionCriteria `xml:"SourceSelectionCriteria" json:"SourceSelectionCriteria"` ExistingObjectReplication ExistingObjectReplication `xml:"ExistingObjectReplication,omitempty" json:"ExistingObjectReplication,omitempty"` } // Validate validates the rule for correctness func (r Rule) Validate() error { if err := r.validateID(); err != nil { return err } if err := r.validateStatus(); err != nil { return err } if err := r.validateFilter(); err != nil { return err } if r.Priority < 0 && r.Status == Enabled { return fmt.Errorf("priority must be set for the rule") } if err := r.validateStatus(); err != nil { return err } return r.ExistingObjectReplication.Validate() } // validateID - checks if ID is valid or not. func (r Rule) validateID() error { // cannot be longer than 255 characters if len(r.ID) > 255 { return fmt.Errorf("ID must be less than 255 characters") } return nil } // validateStatus - checks if status is valid or not. func (r Rule) validateStatus() error { // Status can't be empty if len(r.Status) == 0 { return fmt.Errorf("status cannot be empty") } // Status must be one of Enabled or Disabled if r.Status != Enabled && r.Status != Disabled { return fmt.Errorf("status must be set to either Enabled or Disabled") } return nil } func (r Rule) validateFilter() error { return r.Filter.Validate() } // Prefix - a rule can either have prefix under or under // . This method returns the prefix from the // location where it is available func (r Rule) Prefix() string { if r.Filter.Prefix != "" { return r.Filter.Prefix } return r.Filter.And.Prefix } // Tags - a rule can either have tag under or under // . This method returns all the tags from the // rule in the format tag1=value1&tag2=value2 func (r Rule) Tags() string { ts := []Tag{r.Filter.Tag} if len(r.Filter.And.Tags) != 0 { ts = r.Filter.And.Tags } var buf bytes.Buffer for _, t := range ts { if buf.Len() > 0 { buf.WriteString("&") } buf.WriteString(t.String()) } return buf.String() } // Filter - a filter for a replication configuration Rule. type Filter struct { XMLName xml.Name `xml:"Filter" json:"-"` Prefix string `json:"Prefix,omitempty"` And And `xml:"And,omitempty" json:"And,omitempty"` Tag Tag `xml:"Tag,omitempty" json:"Tag,omitempty"` } // Validate - validates the filter element func (f Filter) Validate() error { // A Filter must have exactly one of Prefix, Tag, or And specified. if !f.And.isEmpty() { if f.Prefix != "" { return errInvalidFilter } if !f.Tag.IsEmpty() { return errInvalidFilter } } if f.Prefix != "" { if !f.Tag.IsEmpty() { return errInvalidFilter } } if !f.Tag.IsEmpty() { if err := f.Tag.Validate(); err != nil { return err } } return nil } // Tag - a tag for a replication configuration Rule filter. type Tag struct { XMLName xml.Name `json:"-"` Key string `xml:"Key,omitempty" json:"Key,omitempty"` Value string `xml:"Value,omitempty" json:"Value,omitempty"` } func (tag Tag) String() string { if tag.IsEmpty() { return "" } return tag.Key + "=" + tag.Value } // IsEmpty returns whether this tag is empty or not. func (tag Tag) IsEmpty() bool { return tag.Key == "" } // Validate checks this tag. func (tag Tag) Validate() error { if len(tag.Key) == 0 || utf8.RuneCountInString(tag.Key) > 128 { return fmt.Errorf("invalid Tag Key") } if utf8.RuneCountInString(tag.Value) > 256 { return fmt.Errorf("invalid Tag Value") } return nil } // Destination - destination in ReplicationConfiguration. type Destination struct { XMLName xml.Name `xml:"Destination" json:"-"` Bucket string `xml:"Bucket" json:"Bucket"` StorageClass string `xml:"StorageClass,omitempty" json:"StorageClass,omitempty"` } // And - a tag to combine a prefix and multiple tags for replication configuration rule. type And struct { XMLName xml.Name `xml:"And,omitempty" json:"-"` Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` Tags []Tag `xml:"Tag,omitempty" json:"Tag,omitempty"` } // isEmpty returns true if Tags field is null func (a And) isEmpty() bool { return len(a.Tags) == 0 && a.Prefix == "" } // Status represents Enabled/Disabled status type Status string // Supported status types const ( Enabled Status = "Enabled" Disabled Status = "Disabled" ) // DeleteMarkerReplication - whether delete markers are replicated - https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html type DeleteMarkerReplication struct { Status Status `xml:"Status" json:"Status"` // should be set to "Disabled" by default } // IsEmpty returns true if DeleteMarkerReplication is not set func (d DeleteMarkerReplication) IsEmpty() bool { return len(d.Status) == 0 } // DeleteReplication - whether versioned deletes are replicated - this // is a MinIO specific extension type DeleteReplication struct { Status Status `xml:"Status" json:"Status"` // should be set to "Disabled" by default } // IsEmpty returns true if DeleteReplication is not set func (d DeleteReplication) IsEmpty() bool { return len(d.Status) == 0 } // ReplicaModifications specifies if replica modification sync is enabled type ReplicaModifications struct { Status Status `xml:"Status" json:"Status"` // should be set to "Enabled" by default } // SourceSelectionCriteria - specifies additional source selection criteria in ReplicationConfiguration. type SourceSelectionCriteria struct { ReplicaModifications ReplicaModifications `xml:"ReplicaModifications" json:"ReplicaModifications"` } // IsValid - checks whether SourceSelectionCriteria is valid or not. func (s SourceSelectionCriteria) IsValid() bool { return s.ReplicaModifications.Status == Enabled || s.ReplicaModifications.Status == Disabled } // Validate source selection criteria func (s SourceSelectionCriteria) Validate() error { if (s == SourceSelectionCriteria{}) { return nil } if !s.IsValid() { return fmt.Errorf("invalid ReplicaModification status") } return nil } // ExistingObjectReplication - whether existing object replication is enabled type ExistingObjectReplication struct { Status Status `xml:"Status"` // should be set to "Disabled" by default } // IsEmpty returns true if DeleteMarkerReplication is not set func (e ExistingObjectReplication) IsEmpty() bool { return len(e.Status) == 0 } // Validate validates whether the status is disabled. func (e ExistingObjectReplication) Validate() error { if e.IsEmpty() { return nil } if e.Status != Disabled && e.Status != Enabled { return fmt.Errorf("invalid ExistingObjectReplication status") } return nil } // TargetMetrics represents inline replication metrics // such as pending, failed and completed bytes in total for a bucket remote target type TargetMetrics struct { // Completed count ReplicatedCount uint64 `json:"replicationCount,omitempty"` // Completed size in bytes ReplicatedSize uint64 `json:"completedReplicationSize,omitempty"` // Bandwidth limit in bytes/sec for this target BandWidthLimitInBytesPerSecond int64 `json:"limitInBits,omitempty"` // Current bandwidth used in bytes/sec for this target CurrentBandwidthInBytesPerSecond float64 `json:"currentBandwidth,omitempty"` // errors seen in replication in last minute, hour and total Failed TimedErrStats `json:"failed,omitempty"` // Deprecated fields // Pending size in bytes PendingSize uint64 `json:"pendingReplicationSize,omitempty"` // Total Replica size in bytes ReplicaSize uint64 `json:"replicaSize,omitempty"` // Failed size in bytes FailedSize uint64 `json:"failedReplicationSize,omitempty"` // Total number of pending operations including metadata updates PendingCount uint64 `json:"pendingReplicationCount,omitempty"` // Total number of failed operations including metadata updates FailedCount uint64 `json:"failedReplicationCount,omitempty"` } // Metrics represents inline replication metrics for a bucket. type Metrics struct { Stats map[string]TargetMetrics // Completed size in bytes across targets ReplicatedSize uint64 `json:"completedReplicationSize,omitempty"` // Total Replica size in bytes across targets ReplicaSize uint64 `json:"replicaSize,omitempty"` // Total Replica counts ReplicaCount int64 `json:"replicaCount,omitempty"` // Total Replicated count ReplicatedCount int64 `json:"replicationCount,omitempty"` // errors seen in replication in last minute, hour and total Errors TimedErrStats `json:"failed,omitempty"` // Total number of entries that are queued for replication QStats InQueueMetric `json:"queued"` // Total number of entries that have replication in progress InProgress InProgressMetric `json:"inProgress"` // Deprecated fields // Total Pending size in bytes across targets PendingSize uint64 `json:"pendingReplicationSize,omitempty"` // Failed size in bytes across targets FailedSize uint64 `json:"failedReplicationSize,omitempty"` // Total number of pending operations including metadata updates across targets PendingCount uint64 `json:"pendingReplicationCount,omitempty"` // Total number of failed operations including metadata updates across targets FailedCount uint64 `json:"failedReplicationCount,omitempty"` } // RStat - has count and bytes for replication metrics type RStat struct { Count float64 `json:"count"` Bytes int64 `json:"bytes"` } // Add two RStat func (r RStat) Add(r1 RStat) RStat { return RStat{ Count: r.Count + r1.Count, Bytes: r.Bytes + r1.Bytes, } } // TimedErrStats holds error stats for a time period type TimedErrStats struct { LastMinute RStat `json:"lastMinute"` LastHour RStat `json:"lastHour"` Totals RStat `json:"totals"` } // Add two TimedErrStats func (te TimedErrStats) Add(o TimedErrStats) TimedErrStats { return TimedErrStats{ LastMinute: te.LastMinute.Add(o.LastMinute), LastHour: te.LastHour.Add(o.LastHour), Totals: te.Totals.Add(o.Totals), } } // ResyncTargetsInfo provides replication target information to resync replicated data. type ResyncTargetsInfo struct { Targets []ResyncTarget `json:"target,omitempty"` } // ResyncTarget provides the replica resources and resetID to initiate resync replication. type ResyncTarget struct { Arn string `json:"arn"` ResetID string `json:"resetid"` StartTime time.Time `json:"startTime,omitempty"` EndTime time.Time `json:"endTime,omitempty"` // Status of resync operation ResyncStatus string `json:"resyncStatus,omitempty"` // Completed size in bytes ReplicatedSize int64 `json:"completedReplicationSize,omitempty"` // Failed size in bytes FailedSize int64 `json:"failedReplicationSize,omitempty"` // Total number of failed operations FailedCount int64 `json:"failedReplicationCount,omitempty"` // Total number of completed operations ReplicatedCount int64 `json:"replicationCount,omitempty"` // Last bucket/object replicated. Bucket string `json:"bucket,omitempty"` Object string `json:"object,omitempty"` } // XferStats holds transfer rate info for uploads/sec type XferStats struct { AvgRate float64 `json:"avgRate"` PeakRate float64 `json:"peakRate"` CurrRate float64 `json:"currRate"` } // Merge two XferStats func (x *XferStats) Merge(x1 XferStats) { x.AvgRate += x1.AvgRate x.PeakRate += x1.PeakRate x.CurrRate += x1.CurrRate } // QStat holds count and bytes for objects in replication queue type QStat struct { Count float64 `json:"count"` Bytes float64 `json:"bytes"` } // Add 2 QStat entries func (q *QStat) Add(q1 QStat) { q.Count += q1.Count q.Bytes += q1.Bytes } // InQueueMetric holds stats for objects in replication queue type InQueueMetric struct { Curr QStat `json:"curr" msg:"cq"` Avg QStat `json:"avg" msg:"aq"` Max QStat `json:"peak" msg:"pq"` } // InProgressMetric holds stats for objects with replication in progress type InProgressMetric InQueueMetric // MetricName name of replication metric type MetricName string const ( // Large is a metric name for large objects >=128MiB Large MetricName = "Large" // Small is a metric name for objects <128MiB size Small MetricName = "Small" // Total is a metric name for total objects Total MetricName = "Total" ) // WorkerStat has stats on number of replication workers type WorkerStat struct { Curr int32 `json:"curr"` Avg float32 `json:"avg"` Max int32 `json:"max"` } // TgtHealth holds health status of a target type TgtHealth struct { Online bool `json:"online"` LastOnline time.Time `json:"lastOnline"` TotalDowntime time.Duration `json:"totalDowntime"` OfflineCount int64 `json:"offlineCount"` } // ReplMRFStats holds stats of MRF backlog saved to disk in the last 5 minutes // and number of entries that failed replication after 3 retries type ReplMRFStats struct { LastFailedCount uint64 `json:"failedCount_last5min"` // Count of unreplicated entries that were dropped after MRF retry limit reached since cluster start. TotalDroppedCount uint64 `json:"droppedCount_since_uptime"` // Bytes of unreplicated entries that were dropped after MRF retry limit reached since cluster start. TotalDroppedBytes uint64 `json:"droppedBytes_since_uptime"` } // ReplQNodeStats holds stats for a node in replication queue type ReplQNodeStats struct { NodeName string `json:"nodeName"` Uptime int64 `json:"uptime"` Workers WorkerStat `json:"workers"` XferStats map[MetricName]XferStats `json:"transferSummary"` TgtXferStats map[string]map[MetricName]XferStats `json:"tgtTransferStats"` QStats InQueueMetric `json:"queueStats"` InProgressStats InProgressMetric `json:"progressStats"` MRFStats ReplMRFStats `json:"mrfStats"` Retries CounterSummary `json:"retries"` Errors CounterSummary `json:"errors"` TgtHealth map[string]TgtHealth `json:"tgtHealth,omitempty"` } // CounterSummary denotes the stats counter summary type CounterSummary struct { // Counted last 1hr Last1hr uint64 `json:"last1hr"` // Counted last 1m Last1m uint64 `json:"last1m"` // Total counted since uptime Total uint64 `json:"total"` } // ReplQueueStats holds stats for replication queue across nodes type ReplQueueStats struct { Nodes []ReplQNodeStats `json:"nodes"` } // Workers returns number of workers across all nodes func (q ReplQueueStats) Workers() (tot WorkerStat) { for _, node := range q.Nodes { tot.Avg += node.Workers.Avg tot.Curr += node.Workers.Curr if tot.Max < node.Workers.Max { tot.Max = node.Workers.Max } } if len(q.Nodes) > 0 { tot.Avg /= float32(len(q.Nodes)) tot.Curr /= int32(len(q.Nodes)) } return tot } // qStatSummary returns cluster level stats for objects in replication queue func (q ReplQueueStats) qStatSummary() InQueueMetric { m := InQueueMetric{} for _, v := range q.Nodes { m.Avg.Add(v.QStats.Avg) m.Curr.Add(v.QStats.Curr) if m.Max.Count < v.QStats.Max.Count { m.Max.Add(v.QStats.Max) } } return m } // inProgressSummary returns cluster level stats for objects with replication in progress func (q ReplQueueStats) inProgressSummary() InProgressMetric { m := InProgressMetric{} for _, v := range q.Nodes { m.Avg.Add(v.InProgressStats.Avg) m.Curr.Add(v.InProgressStats.Curr) if m.Max.Count < v.InProgressStats.Max.Count { m.Max.Add(v.InProgressStats.Max) } } return m } // ReplQStats holds stats for objects in replication queue type ReplQStats struct { Uptime int64 `json:"uptime"` Workers WorkerStat `json:"workers"` XferStats map[MetricName]XferStats `json:"xferStats"` TgtXferStats map[string]map[MetricName]XferStats `json:"tgtXferStats"` QStats InQueueMetric `json:"qStats"` InProgressStats InProgressMetric `json:"progressStats"` MRFStats ReplMRFStats `json:"mrfStats"` Retries CounterSummary `json:"retries"` Errors CounterSummary `json:"errors"` } // QStats returns cluster level stats for objects in replication queue func (q ReplQueueStats) QStats() (r ReplQStats) { r.QStats = q.qStatSummary() r.InProgressStats = q.inProgressSummary() r.XferStats = make(map[MetricName]XferStats) r.TgtXferStats = make(map[string]map[MetricName]XferStats) r.Workers = q.Workers() for _, node := range q.Nodes { for arn := range node.TgtXferStats { xmap, ok := node.TgtXferStats[arn] if !ok { xmap = make(map[MetricName]XferStats) } for m, v := range xmap { st, ok := r.XferStats[m] if !ok { st = XferStats{} } st.AvgRate += v.AvgRate st.CurrRate += v.CurrRate st.PeakRate = math.Max(st.PeakRate, v.PeakRate) if _, ok := r.TgtXferStats[arn]; !ok { r.TgtXferStats[arn] = make(map[MetricName]XferStats) } r.TgtXferStats[arn][m] = st } } for k, v := range node.XferStats { st, ok := r.XferStats[k] if !ok { st = XferStats{} } st.AvgRate += v.AvgRate st.CurrRate += v.CurrRate st.PeakRate = math.Max(st.PeakRate, v.PeakRate) r.XferStats[k] = st } r.MRFStats.LastFailedCount += node.MRFStats.LastFailedCount r.MRFStats.TotalDroppedCount += node.MRFStats.TotalDroppedCount r.MRFStats.TotalDroppedBytes += node.MRFStats.TotalDroppedBytes r.Retries.Last1hr += node.Retries.Last1hr r.Retries.Last1m += node.Retries.Last1m r.Retries.Total += node.Retries.Total r.Errors.Last1hr += node.Errors.Last1hr r.Errors.Last1m += node.Errors.Last1m r.Errors.Total += node.Errors.Total r.Uptime += node.Uptime } if len(q.Nodes) > 0 { r.Uptime /= int64(len(q.Nodes)) // average uptime } return r } // MetricsV2 represents replication metrics for a bucket. type MetricsV2 struct { Uptime int64 `json:"uptime"` CurrentStats Metrics `json:"currStats"` QueueStats ReplQueueStats `json:"queueStats"` DowntimeInfo map[string]DowntimeInfo `json:"downtimeInfo"` } // DowntimeInfo represents the downtime info type DowntimeInfo struct { Duration Stat `json:"duration"` Count Stat `json:"count"` } // Stat represents the aggregates type Stat struct { Total int64 `json:"total"` Avg int64 `json:"avg"` Max int64 `json:"max"` } minio-go-7.0.97/pkg/replication/replication_test.go000066400000000000000000000242011510244170000223160ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package replication import ( "testing" ) // Tests replication rule addition. func TestAddReplicationRule(t *testing.T) { testCases := []struct { cfg Config opts Options expectedErr string }{ { // test case :1 cfg: Config{}, opts: Options{ ID: "xyz.id", Prefix: "abc/", RuleStatus: "enable", Priority: "3", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket", }, expectedErr: "", }, { // test case :2 cfg: Config{}, opts: Options{ ID: "", Prefix: "abc/", RuleStatus: "", Priority: "3", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket", }, expectedErr: "rule state should be either [enable|disable]", }, { // test case :3 cfg: Config{Rules: []Rule{{Priority: 1}}}, opts: Options{ ID: "xyz.id", Prefix: "abc/", RuleStatus: "enable", Priority: "1", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket", }, expectedErr: "priority must be unique. Replication configuration already has a rule with this priority", }, { // test case :4 cfg: Config{}, opts: Options{ ID: "xyz.id", Prefix: "abc/", RuleStatus: "enable", Priority: "3", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:minio:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket", }, expectedErr: "destination bucket needs to be in Arn format", }, { // test case :5 cfg: Config{}, opts: Options{ ID: "xyz.id", Prefix: "abc/", RuleStatus: "enable", Priority: "3", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:destbucket", }, expectedErr: "destination bucket needs to be in Arn format", }, { // test case :6 cfg: Config{Role: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:targetbucket"}, opts: Options{ ID: "xyz.id", Prefix: "abc/", RuleStatus: "enable", Priority: "3", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket", }, expectedErr: "", }, { // test case :7 cfg: Config{}, opts: Options{ ID: "xyz.id", Prefix: "abc/", RuleStatus: "enable", Priority: "3", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:aws:s3:::destbucket", }, expectedErr: "", }, { // test case :8 cfg: Config{ Rules: []Rule{ { ID: "xyz.id", Destination: Destination{ Bucket: "arn:aws:s3:::destbucket", }, }, }, }, opts: Options{ ID: "xyz.id", Prefix: "abc/", RuleStatus: "enable", Priority: "1", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket", }, expectedErr: "a rule exists with this ID", }, } for i, testCase := range testCases { cfg := testCase.cfg err := cfg.AddRule(testCase.opts) if err != nil && testCase.expectedErr != err.Error() { t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, err) } if err == nil && testCase.expectedErr != "" { t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, err) } } } // Tests replication rule edits. func TestEditReplicationRule(t *testing.T) { testCases := []struct { cfg Config opts Options expectedErr string }{ { // test case :1 edit a rule in older config with remote ARN in destination bucket cfg: Config{ Role: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket", Rules: []Rule{{ ID: "xyz.id", Priority: 1, Filter: Filter{Prefix: "xyz/"}, Destination: Destination{Bucket: "arn:aws:s3:::destbucket"}, }}, }, opts: Options{ ID: "xyz.id", Prefix: "abc/", RuleStatus: "enable", Priority: "3", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket", }, expectedErr: "", }, { // test case :2 mismatched rule id cfg: Config{ Role: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket", Rules: []Rule{{ ID: "xyz.id2", Priority: 1, Filter: Filter{Prefix: "xyz/"}, Destination: Destination{Bucket: "arn:aws:s3:::destbucket"}, }}, }, opts: Options{ ID: "xyz.id", Prefix: "abc/", RuleStatus: "enable", Priority: "3", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket", }, expectedErr: "rule with ID xyz.id not found in replication configuration", }, { // test case :3 missing rule id cfg: Config{ Role: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket", Rules: []Rule{{ ID: "xyz.id2", Priority: 1, Filter: Filter{Prefix: "xyz/"}, Destination: Destination{Bucket: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket"}, }}, }, opts: Options{ Prefix: "abc/", RuleStatus: "enable", Priority: "3", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket", }, expectedErr: "rule ID missing", }, { // test case :4 different destination bucket cfg: Config{ Role: "", Rules: []Rule{{ ID: "xyz.id", Priority: 1, Filter: Filter{Prefix: "xyz/"}, Destination: Destination{Bucket: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket"}, }}, }, opts: Options{ ID: "xyz.id", Prefix: "abc/", RuleStatus: "enable", Priority: "3", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:aws:s3:::differentbucket", }, expectedErr: "invalid destination bucket for this rule", }, { // test case :5 invalid destination bucket arn format cfg: Config{ Role: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket", Rules: []Rule{{ ID: "xyz.id", Priority: 1, Filter: Filter{Prefix: "xyz/"}, Destination: Destination{Bucket: "arn:aws:s3:::destbucket"}, }}, }, opts: Options{ ID: "xyz.id", Prefix: "abc/", RuleStatus: "enable", Priority: "3", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:destbucket", }, expectedErr: "destination bucket needs to be in Arn format", }, { // test case :6 invalid rule status cfg: Config{ Rules: []Rule{{ ID: "xyz.id", Priority: 1, Filter: Filter{Prefix: "xyz/"}, Destination: Destination{Bucket: "arn:aws:s3:::destbucket"}, }}, }, opts: Options{ ID: "xyz.id", Prefix: "abc/", RuleStatus: "xx", Priority: "3", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:aws:s3:::destbucket", }, expectedErr: "rule state should be either [enable|disable]", }, { // test case :7 another rule has same priority cfg: Config{ Rules: []Rule{ { ID: "xyz.id", Priority: 0, Filter: Filter{Prefix: "xyz/"}, Destination: Destination{Bucket: "arn:aws:s3:::destbucket"}, }, { ID: "xyz.id2", Priority: 1, Filter: Filter{Prefix: "xyz/"}, Destination: Destination{Bucket: "arn:aws:s3:::destbucket"}, }, }, }, opts: Options{ ID: "xyz.id", Prefix: "abc/", RuleStatus: "disable", Priority: "1", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:aws:s3:::destbucket", }, expectedErr: "priority must be unique. Replication configuration already has a rule with this priority", }, { // test case :8 ; edit a rule in older config cfg: Config{ Role: "arn:minio:replication:eu-west-1:c5acb6ac-9918-4dc6-8534-6244ed1a611a:destbucket", Rules: []Rule{{ ID: "xyz.id", Priority: 1, Filter: Filter{Prefix: "xyz/"}, Destination: Destination{Bucket: "arn:aws:s3:::destbucket"}, }}, }, opts: Options{ ID: "xyz.id", Prefix: "abc/", RuleStatus: "enable", Priority: "3", TagString: "k1=v1&k2=v2", StorageClass: "STANDARD", DestBucket: "arn:aws:s3:::destbucket", }, expectedErr: "", }, } for i, testCase := range testCases { cfg := testCase.cfg err := cfg.EditRule(testCase.opts) if err != nil && testCase.expectedErr != err.Error() { t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, err) } if err == nil && testCase.expectedErr != "" { t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, err) } } } minio-go-7.0.97/pkg/s3utils/000077500000000000000000000000001510244170000155155ustar00rootroot00000000000000minio-go-7.0.97/pkg/s3utils/utils.go000066400000000000000000000367251510244170000172210ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package s3utils import ( "bytes" "encoding/hex" "errors" "net" "net/url" "regexp" "sort" "strings" "unicode/utf8" ) // Sentinel URL is the default url value which is invalid. var sentinelURL = url.URL{} // IsValidDomain validates if input string is a valid domain name. func IsValidDomain(host string) bool { // See RFC 1035, RFC 3696. host = strings.TrimSpace(host) if len(host) == 0 || len(host) > 255 { return false } // host cannot start or end with "-" if host[len(host)-1:] == "-" || host[:1] == "-" { return false } // host cannot start or end with "_" if host[len(host)-1:] == "_" || host[:1] == "_" { return false } // host cannot start with a "." if host[:1] == "." { return false } // All non alphanumeric characters are invalid. if strings.ContainsAny(host, "`~!@#$%^&*()+={}[]|\\\"';:> 1 { return parts[1] } parts = amazonS3HostFIPS.FindStringSubmatch(endpointURL.Hostname()) if len(parts) > 1 { return parts[1] } parts = amazonS3HostDualStack.FindStringSubmatch(endpointURL.Hostname()) if len(parts) > 1 { return parts[1] } parts = amazonS3HostHyphen.FindStringSubmatch(endpointURL.Hostname()) if len(parts) > 1 { return parts[1] } parts = amazonS3ChinaHost.FindStringSubmatch(endpointURL.Hostname()) if len(parts) > 1 { return parts[1] } parts = amazonS3ChinaHostDualStack.FindStringSubmatch(endpointURL.Hostname()) if len(parts) > 1 { return parts[1] } parts = amazonS3HostPrivateLink.FindStringSubmatch(endpointURL.Hostname()) if len(parts) > 1 { return parts[1] } parts = amazonS3HostExpress.FindStringSubmatch(endpointURL.Hostname()) if len(parts) > 1 { return parts[1] } parts = amazonS3HostExpressControl.FindStringSubmatch(endpointURL.Hostname()) if len(parts) > 1 { return parts[1] } parts = amazonS3HostDot.FindStringSubmatch(endpointURL.Hostname()) if len(parts) > 1 { if strings.HasPrefix(parts[1], "xpress-") { return "" } if strings.HasPrefix(parts[1], "dualstack.") || strings.HasPrefix(parts[1], "control.") || strings.HasPrefix(parts[1], "website-") { return "" } return parts[1] } return "" } // IsAliyunOSSEndpoint - Match if it is exactly Aliyun OSS endpoint. func IsAliyunOSSEndpoint(endpointURL url.URL) bool { return strings.HasSuffix(endpointURL.Hostname(), "aliyuncs.com") } // IsAmazonExpressRegionalEndpoint Match if the endpoint is S3 Express regional endpoint. func IsAmazonExpressRegionalEndpoint(endpointURL url.URL) bool { return amazonS3HostExpressControl.MatchString(endpointURL.Hostname()) } // IsAmazonExpressZonalEndpoint Match if the endpoint is S3 Express zonal endpoint. func IsAmazonExpressZonalEndpoint(endpointURL url.URL) bool { return amazonS3HostExpress.MatchString(endpointURL.Hostname()) } // IsAmazonEndpoint - Match if it is exactly Amazon S3 endpoint. func IsAmazonEndpoint(endpointURL url.URL) bool { if endpointURL.Hostname() == "s3-external-1.amazonaws.com" || endpointURL.Hostname() == "s3.amazonaws.com" { return true } return GetRegionFromURL(endpointURL) != "" } // IsAmazonGovCloudEndpoint - Match if it is exactly Amazon S3 GovCloud endpoint. func IsAmazonGovCloudEndpoint(endpointURL url.URL) bool { if endpointURL == sentinelURL { return false } return (endpointURL.Host == "s3-us-gov-west-1.amazonaws.com" || endpointURL.Host == "s3-us-gov-east-1.amazonaws.com" || IsAmazonFIPSGovCloudEndpoint(endpointURL)) } // IsAmazonFIPSGovCloudEndpoint - match if the endpoint is FIPS and GovCloud. func IsAmazonFIPSGovCloudEndpoint(endpointURL url.URL) bool { if endpointURL == sentinelURL { return false } return IsAmazonFIPSEndpoint(endpointURL) && strings.Contains(endpointURL.Hostname(), "us-gov-") } // IsAmazonFIPSEndpoint - Match if it is exactly Amazon S3 FIPS endpoint. // See https://aws.amazon.com/compliance/fips. func IsAmazonFIPSEndpoint(endpointURL url.URL) bool { if endpointURL == sentinelURL { return false } return strings.HasPrefix(endpointURL.Hostname(), "s3-fips") && strings.HasSuffix(endpointURL.Hostname(), ".amazonaws.com") } // IsAmazonPrivateLinkEndpoint - Match if it is exactly Amazon S3 PrivateLink interface endpoint // See https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html. func IsAmazonPrivateLinkEndpoint(endpointURL url.URL) bool { if endpointURL == sentinelURL { return false } return amazonS3HostPrivateLink.MatchString(endpointURL.Hostname()) } // IsGoogleEndpoint - Match if it is exactly Google cloud storage endpoint. func IsGoogleEndpoint(endpointURL url.URL) bool { if endpointURL == sentinelURL { return false } return endpointURL.Hostname() == "storage.googleapis.com" } // Expects ascii encoded strings - from output of urlEncodePath func percentEncodeSlash(s string) string { return strings.ReplaceAll(s, "/", "%2F") } // QueryEncode - encodes query values in their URL encoded form. In // addition to the percent encoding performed by urlEncodePath() used // here, it also percent encodes '/' (forward slash) func QueryEncode(v url.Values) string { if v == nil { return "" } var buf bytes.Buffer keys := make([]string, 0, len(v)) for k := range v { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { vs := v[k] prefix := percentEncodeSlash(EncodePath(k)) + "=" for _, v := range vs { if buf.Len() > 0 { buf.WriteByte('&') } buf.WriteString(prefix) buf.WriteString(percentEncodeSlash(EncodePath(v))) } } return buf.String() } // if object matches reserved string, no need to encode them var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") // EncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences // // This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8 // non english characters cannot be parsed due to the nature in which url.Encode() is written // // This function on the other hand is a direct replacement for url.Encode() technique to support // pretty much every UTF-8 character. func EncodePath(pathName string) string { if reservedObjectNames.MatchString(pathName) { return pathName } var encodedPathname strings.Builder for _, s := range pathName { if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) encodedPathname.WriteRune(s) continue } switch s { case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) encodedPathname.WriteRune(s) continue default: l := utf8.RuneLen(s) if l < 0 { // if utf8 cannot convert return the same string as is return pathName } u := make([]byte, l) utf8.EncodeRune(u, s) for _, r := range u { hex := hex.EncodeToString([]byte{r}) encodedPathname.WriteString("%" + strings.ToUpper(hex)) } } } return encodedPathname.String() } // We support '.' with bucket names but we fallback to using path // style requests instead for such buckets. var ( validBucketName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9\.\-\_\:]{1,61}[A-Za-z0-9]$`) validBucketNameStrict = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`) validBucketNameS3Express = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]--[a-z0-9]{3,7}-az[1-6]--x-s3$`) ipAddress = regexp.MustCompile(`^(\d+\.){3}\d+$`) ) // Common checker for both stricter and basic validation. func checkBucketNameCommon(bucketName string, strict bool) (err error) { if strings.TrimSpace(bucketName) == "" { return errors.New("Bucket name cannot be empty") } if len(bucketName) < 3 { return errors.New("Bucket name cannot be shorter than 3 characters") } if len(bucketName) > 63 { return errors.New("Bucket name cannot be longer than 63 characters") } if ipAddress.MatchString(bucketName) { return errors.New("Bucket name cannot be an ip address") } if strings.Contains(bucketName, "..") || strings.Contains(bucketName, ".-") || strings.Contains(bucketName, "-.") { return errors.New("Bucket name contains invalid characters") } if strict { if !validBucketNameStrict.MatchString(bucketName) { err = errors.New("Bucket name contains invalid characters") } return err } if !validBucketName.MatchString(bucketName) { err = errors.New("Bucket name contains invalid characters") } return err } // CheckValidBucketName - checks if we have a valid input bucket name. func CheckValidBucketName(bucketName string) (err error) { return checkBucketNameCommon(bucketName, false) } // IsS3ExpressBucket is S3 express bucket? func IsS3ExpressBucket(bucketName string) bool { return CheckValidBucketNameS3Express(bucketName) == nil } // CheckValidBucketNameS3Express - checks if we have a valid input bucket name for S3 Express. func CheckValidBucketNameS3Express(bucketName string) (err error) { if strings.TrimSpace(bucketName) == "" { return errors.New("Bucket name cannot be empty for S3 Express") } if len(bucketName) < 3 { return errors.New("Bucket name cannot be shorter than 3 characters for S3 Express") } if len(bucketName) > 63 { return errors.New("Bucket name cannot be longer than 63 characters for S3 Express") } // Check if the bucket matches the regex if !validBucketNameS3Express.MatchString(bucketName) { return errors.New("Bucket name contains invalid characters") } // Extract bucket name (before ----x-s3) parts := strings.Split(bucketName, "--") if len(parts) != 3 || parts[2] != "x-s3" { return errors.New("Bucket name pattern is wrong 'x-s3'") } bucketName = parts[0] // Additional validation for bucket name // 1. No consecutive periods or hyphens if strings.Contains(bucketName, "..") || strings.Contains(bucketName, "--") { return errors.New("Bucket name contains invalid characters") } // 2. No period-hyphen or hyphen-period if strings.Contains(bucketName, ".-") || strings.Contains(bucketName, "-.") { return errors.New("Bucket name has unexpected format or contains invalid characters") } // 3. No IP address format (e.g., 192.168.0.1) if ipAddress.MatchString(bucketName) { return errors.New("Bucket name cannot be an ip address") } return nil } // CheckValidBucketNameStrict - checks if we have a valid input bucket name. // This is a stricter version. // - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html func CheckValidBucketNameStrict(bucketName string) (err error) { return checkBucketNameCommon(bucketName, true) } // CheckValidObjectNamePrefix - checks if we have a valid input object name prefix. // - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html func CheckValidObjectNamePrefix(objectName string) error { if len(objectName) > 1024 { return errors.New("Object name cannot be longer than 1024 characters") } if !utf8.ValidString(objectName) { return errors.New("Object name with non UTF-8 strings are not supported") } return nil } // CheckValidObjectName - checks if we have a valid input object name. // - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html func CheckValidObjectName(objectName string) error { if strings.TrimSpace(objectName) == "" { return errors.New("Object name cannot be empty") } return CheckValidObjectNamePrefix(objectName) } minio-go-7.0.97/pkg/s3utils/utils_test.go000066400000000000000000000450471510244170000202550ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package s3utils import ( "errors" "net/url" "testing" ) // Tests get region from host URL. func TestGetRegionFromURL(t *testing.T) { testCases := []struct { u string expectedRegion string }{ {u: "storage.googleapis.com", expectedRegion: ""}, {u: "s3.cn-north-1.amazonaws.com.cn", expectedRegion: "cn-north-1"}, {u: "s3.dualstack.cn-north-1.amazonaws.com.cn", expectedRegion: "cn-north-1"}, {u: "s3.cn-northwest-1.amazonaws.com.cn", expectedRegion: "cn-northwest-1"}, {u: "s3.dualstack.cn-northwest-1.amazonaws.com.cn", expectedRegion: "cn-northwest-1"}, {u: "s3-fips-us-gov-west-1.amazonaws.com", expectedRegion: "us-gov-west-1"}, {u: "s3-fips.us-gov-west-1.amazonaws.com", expectedRegion: "us-gov-west-1"}, {u: "s3-fips.us-gov-east-1.amazonaws.com", expectedRegion: "us-gov-east-1"}, {u: "s3-us-gov-west-1.amazonaws.com", expectedRegion: "us-gov-west-1"}, {u: "192.168.1.1", expectedRegion: ""}, {u: "s3-eu-west-1.amazonaws.com", expectedRegion: "eu-west-1"}, {u: "s3.eu-west-1.amazonaws.com", expectedRegion: "eu-west-1"}, {u: "s3.dualstack.eu-west-1.amazonaws.com", expectedRegion: "eu-west-1"}, {u: "s3.amazonaws.com", expectedRegion: ""}, {u: "s3-external-1.amazonaws.com", expectedRegion: ""}, {u: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.us-west-2.amazonaws.com", expectedRegion: ""}, {u: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.amazonaws.com", expectedRegion: ""}, {u: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.amazonaws.com.cn", expectedRegion: ""}, {u: "bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com", expectedRegion: "us-east-1"}, {u: "accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com", expectedRegion: "us-east-1"}, {u: "s3-fips.us-east-1.amazonaws.com", expectedRegion: "us-east-1"}, {u: "s3-fips.dualstack.us-west-1.amazonaws.com", expectedRegion: "us-west-1"}, {u: "s3express-usw2-az1.us-west-2.amazonaws.com", expectedRegion: "us-west-2"}, {u: "s3express-use1-az5.us-east-1.amazonaws.com", expectedRegion: "us-east-1"}, {u: "s3express-apne1-az4.ap-northeast-1.amazonaws.com", expectedRegion: "ap-northeast-1"}, {u: "s3express-euc1-az2.eu-central-1.amazonaws.com", expectedRegion: "eu-central-1"}, {u: "s3express-usgw1-az3.us-gov-west-1.amazonaws.com", expectedRegion: "us-gov-west-1"}, {u: "s3express-control.us-west-2.amazonaws.com", expectedRegion: "us-west-2"}, // Test cases with port numbers. {u: "storage.googleapis.com:80", expectedRegion: ""}, {u: "s3.cn-north-1.amazonaws.com.cn:80", expectedRegion: "cn-north-1"}, {u: "s3.dualstack.cn-north-1.amazonaws.com.cn:80", expectedRegion: "cn-north-1"}, {u: "s3.cn-northwest-1.amazonaws.com.cn:80", expectedRegion: "cn-northwest-1"}, {u: "s3.dualstack.cn-northwest-1.amazonaws.com.cn:80", expectedRegion: "cn-northwest-1"}, {u: "s3-fips-us-gov-west-1.amazonaws.com:80", expectedRegion: "us-gov-west-1"}, {u: "s3-fips.us-gov-west-1.amazonaws.com:80", expectedRegion: "us-gov-west-1"}, {u: "s3-fips.us-gov-east-1.amazonaws.com:80", expectedRegion: "us-gov-east-1"}, {u: "s3-us-gov-west-1.amazonaws.com:80", expectedRegion: "us-gov-west-1"}, {u: "s3-eu-west-1.amazonaws.com:80", expectedRegion: "eu-west-1"}, {u: "s3.eu-west-1.amazonaws.com:80", expectedRegion: "eu-west-1"}, {u: "s3.dualstack.eu-west-1.amazonaws.com:80", expectedRegion: "eu-west-1"}, {u: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.us-west-2.amazonaws.com:80", expectedRegion: ""}, {u: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.amazonaws.com:80", expectedRegion: ""}, {u: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.amazonaws.com.cn:80", expectedRegion: ""}, {u: "bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com:80", expectedRegion: "us-east-1"}, {u: "accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com:80", expectedRegion: "us-east-1"}, {u: "s3-fips.us-east-1.amazonaws.com:80", expectedRegion: "us-east-1"}, {u: "s3-fips.dualstack.us-west-1.amazonaws.com:80", expectedRegion: "us-west-1"}, // No region found. {u: "192.168.1.1:80", expectedRegion: ""}, {u: "s3express-usw2-az7.us-west-2.amazonaws.com", expectedRegion: ""}, {u: "invalid-endpoint.com", expectedRegion: ""}, {u: "s3.amazonaws.com:80", expectedRegion: ""}, {u: "s3-external-1.amazonaws.com:80", expectedRegion: ""}, } for i, testCase := range testCases { region := GetRegionFromURL(url.URL{Host: testCase.u}) if testCase.expectedRegion != region { t.Errorf("Test %d: Expected region %s, got %s", i+1, testCase.expectedRegion, region) } } } // Tests for 'isValidDomain(host string) bool'. func TestIsValidDomain(t *testing.T) { testCases := []struct { // Input. host string // Expected result. result bool }{ {"s3.amazonaws.com", true}, {"s3.cn-north-1.amazonaws.com.cn", true}, {"s3.cn-northwest-1.amazonaws.com.cn", true}, {"s3.amazonaws.com_", false}, {"%$$$", false}, {"s3.amz.test.com", true}, {"s3.%%", false}, {"localhost", true}, {"localhost.", true}, // http://www.dns-sd.org/trailingdotsindomainnames.html {"-localhost", false}, {"", false}, {"\n \t", false}, {" ", false}, } for i, testCase := range testCases { result := IsValidDomain(testCase.host) if testCase.result != result { t.Errorf("Test %d: Expected isValidDomain test to be '%v', but found '%v' instead", i+1, testCase.result, result) } } } // Tests validate IP address validator. func TestIsValidIP(t *testing.T) { testCases := []struct { // Input. ip string // Expected result. result bool }{ {"192.168.1.1", true}, {"192.168.1", false}, {"192.168.1.1.1", false}, {"-192.168.1.1", false}, {"260.192.1.1", false}, } for i, testCase := range testCases { result := IsValidIP(testCase.ip) if testCase.result != result { t.Errorf("Test %d: Expected isValidIP to be '%v' for input \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.ip, result) } } } // Tests validate virtual host validator. func TestIsVirtualHostSupported(t *testing.T) { testCases := []struct { url string bucket string // Expeceted result. result bool }{ {"https://s3.amazonaws.com", "my-bucket", true}, {"https://s3.cn-north-1.amazonaws.com.cn", "my-bucket", true}, {"https://s3.amazonaws.com", "my-bucket.", false}, {"https://amazons3.amazonaws.com", "my-bucket.", false}, {"https://storage.googleapis.com/", "my-bucket", true}, {"https://mystorage.googleapis.com/", "my-bucket", false}, } for i, testCase := range testCases { u, err := url.Parse(testCase.url) if err != nil { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err) } result := IsVirtualHostSupported(*u, testCase.bucket) if testCase.result != result { t.Errorf("Test %d: Expected isVirtualHostSupported to be '%v' for input url \"%s\" and bucket \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.url, testCase.bucket, result) } } } // Tests validate Amazon endpoint validator. func TestIsAmazonEndpoint(t *testing.T) { testCases := []struct { url string // Expected result. result bool }{ {"https://192.168.1.1", false}, {"192.168.1.1", false}, {"http://storage.googleapis.com", false}, {"https://storage.googleapis.com", false}, {"storage.googleapis.com", false}, {"s3.amazonaws.com", false}, {"https://amazons3.amazonaws.com", false}, {"-192.168.1.1", false}, {"260.192.1.1", false}, {"https://s3-.amazonaws.com", false}, {"https://s3..amazonaws.com", false}, {"https://s3.dualstack.us-west-1.amazonaws.com.cn", false}, {"https://s3..us-west-1.amazonaws.com.cn", false}, // valid inputs. {"https://s3.amazonaws.com", true}, {"https://s3-external-1.amazonaws.com", true}, {"https://s3.cn-north-1.amazonaws.com.cn", true}, {"https://s3-us-west-1.amazonaws.com", true}, {"https://s3.us-west-1.amazonaws.com", true}, {"https://s3.dualstack.us-west-1.amazonaws.com", true}, {"https://bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com", true}, {"https://accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com", true}, {"https://s3express-usw2-az1.us-west-2.amazonaws.com", true}, } for i, testCase := range testCases { u, err := url.Parse(testCase.url) if err != nil { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err) } result := IsAmazonEndpoint(*u) if testCase.result != result { t.Errorf("Test %d: Expected isAmazonEndpoint to be '%v' for input \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.url, result) } } } // Tests validate Google Cloud end point validator. func TestIsGoogleEndpoint(t *testing.T) { testCases := []struct { url string // Expected result. result bool }{ {"192.168.1.1", false}, {"https://192.168.1.1", false}, {"s3.amazonaws.com", false}, {"http://s3.amazonaws.com", false}, {"https://s3.amazonaws.com", false}, {"https://s3.cn-north-1.amazonaws.com.cn", false}, {"-192.168.1.1", false}, {"260.192.1.1", false}, // valid inputs. {"http://storage.googleapis.com", true}, {"https://storage.googleapis.com", true}, {"http://storage.googleapis.com:80", true}, {"https://storage.googleapis.com:443", true}, } for i, testCase := range testCases { u, err := url.Parse(testCase.url) if err != nil { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err) } result := IsGoogleEndpoint(*u) if testCase.result != result { t.Errorf("Test %d: Expected isGoogleEndpoint to be '%v' for input \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.url, result) } } } func TestPercentEncodeSlash(t *testing.T) { testCases := []struct { input string output string }{ {"test123", "test123"}, {"abc,+_1", "abc,+_1"}, {"%40prefix=test%40123", "%40prefix=test%40123"}, {"key1=val1/val2", "key1=val1%2Fval2"}, {"%40prefix=test%40123/", "%40prefix=test%40123%2F"}, } for i, testCase := range testCases { receivedOutput := percentEncodeSlash(testCase.input) if testCase.output != receivedOutput { t.Errorf( "Test %d: Input: \"%s\" --> Expected percentEncodeSlash to return \"%s\", but it returned \"%s\" instead!", i+1, testCase.input, testCase.output, receivedOutput, ) } } } // Tests validate the query encoder. func TestQueryEncode(t *testing.T) { testCases := []struct { queryKey string valueToEncode []string // Expected result. result string }{ {"prefix", []string{"test@123", "test@456"}, "prefix=test%40123&prefix=test%40456"}, {"@prefix", []string{"test@123"}, "%40prefix=test%40123"}, {"@prefix", []string{"a/b/c/"}, "%40prefix=a%2Fb%2Fc%2F"}, {"prefix", []string{"test#123"}, "prefix=test%23123"}, {"prefix#", []string{"test#123"}, "prefix%23=test%23123"}, {"prefix", []string{"test123"}, "prefix=test123"}, {"prefix", []string{"test本語123", "test123"}, "prefix=test%E6%9C%AC%E8%AA%9E123&prefix=test123"}, } for i, testCase := range testCases { urlValues := make(url.Values) for _, valueToEncode := range testCase.valueToEncode { urlValues.Add(testCase.queryKey, valueToEncode) } result := QueryEncode(urlValues) if testCase.result != result { t.Errorf("Test %d: Expected queryEncode result to be \"%s\", but found it to be \"%s\" instead", i+1, testCase.result, result) } } } // Tests validate the URL path encoder. func TestEncodePath(t *testing.T) { testCases := []struct { // Input. inputStr string // Expected result. result string }{ {"thisisthe%url", "thisisthe%25url"}, {"本語", "%E6%9C%AC%E8%AA%9E"}, {"本語.1", "%E6%9C%AC%E8%AA%9E.1"}, {">123", "%3E123"}, {"myurl#link", "myurl%23link"}, {"space in url", "space%20in%20url"}, {"url+path", "url%2Bpath"}, {"url/path", "url/path"}, } for i, testCase := range testCases { result := EncodePath(testCase.inputStr) if testCase.result != result { t.Errorf("Test %d: Expected queryEncode result to be \"%s\", but found it to be \"%s\" instead", i+1, testCase.result, result) } } } // Tests validate the bucket name validator. func TestIsValidBucketName(t *testing.T) { testCases := []struct { // Input. bucketName string // Expected result. err error // Flag to indicate whether test should Pass. shouldPass bool }{ {".mybucket", errors.New("Bucket name contains invalid characters"), false}, {"$mybucket", errors.New("Bucket name contains invalid characters"), false}, {"mybucket-", errors.New("Bucket name contains invalid characters"), false}, {"my", errors.New("Bucket name cannot be shorter than 3 characters"), false}, {"", errors.New("Bucket name cannot be empty"), false}, {"my..bucket", errors.New("Bucket name contains invalid characters"), false}, {"my.-bucket", errors.New("Bucket name contains invalid characters"), false}, {"my-.bucket", errors.New("Bucket name contains invalid characters"), false}, {"192.168.1.168", errors.New("Bucket name cannot be an ip address"), false}, {":bucketname", errors.New("Bucket name contains invalid characters"), false}, {"_bucketName", errors.New("Bucket name contains invalid characters"), false}, {"my.bucket.com", nil, true}, {"my-bucket", nil, true}, {"123my-bucket", nil, true}, {"Mybucket", nil, true}, {"My_bucket", nil, true}, {"My:bucket", nil, true}, } for i, testCase := range testCases { err := CheckValidBucketName(testCase.bucketName) if err != nil && testCase.shouldPass { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) } if err == nil && !testCase.shouldPass { t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) } // Failed as expected, but does it fail for the expected reason. if err != nil && !testCase.shouldPass { if err.Error() != testCase.err.Error() { t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) } } } } // Tests validate the bucket name validator stricter. func TestIsValidBucketNameStrict(t *testing.T) { testCases := []struct { // Input. bucketName string // Expected result. err error // Flag to indicate whether test should Pass. shouldPass bool }{ {".mybucket", errors.New("Bucket name contains invalid characters"), false}, {"$mybucket", errors.New("Bucket name contains invalid characters"), false}, {"mybucket-", errors.New("Bucket name contains invalid characters"), false}, {"my", errors.New("Bucket name cannot be shorter than 3 characters"), false}, {"", errors.New("Bucket name cannot be empty"), false}, {"my..bucket", errors.New("Bucket name contains invalid characters"), false}, {"my.-bucket", errors.New("Bucket name contains invalid characters"), false}, {"my-.bucket", errors.New("Bucket name contains invalid characters"), false}, {"192.168.1.168", errors.New("Bucket name cannot be an ip address"), false}, {"Mybucket", errors.New("Bucket name contains invalid characters"), false}, {"my.bucket.com", nil, true}, {"my-bucket", nil, true}, {"123my-bucket", nil, true}, } for i, testCase := range testCases { err := CheckValidBucketNameStrict(testCase.bucketName) if err != nil && testCase.shouldPass { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) } if err == nil && !testCase.shouldPass { t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) } // Failed as expected, but does it fail for the expected reason. if err != nil && !testCase.shouldPass { if err.Error() != testCase.err.Error() { t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) } } } } func TestIsAmazonPrivateLinkEndpoint(t *testing.T) { testCases := []struct { url string // Expected result. result bool }{ {"https://192.168.1.1", false}, {"192.168.1.1", false}, {"http://storage.googleapis.com", false}, {"https://storage.googleapis.com", false}, {"storage.googleapis.com", false}, {"s3.amazonaws.com", false}, {"https://amazons3.amazonaws.com", false}, {"-192.168.1.1", false}, {"260.192.1.1", false}, {"https://s3-.amazonaws.com", false}, {"https://s3..amazonaws.com", false}, {"https://s3.dualstack.us-west-1.amazonaws.com.cn", false}, {"https://s3..us-west-1.amazonaws.com.cn", false}, {"https://s3.amazonaws.com", false}, {"https://s3-external-1.amazonaws.com", false}, {"https://s3.cn-north-1.amazonaws.com.cn", false}, {"https://s3-us-west-1.amazonaws.com", false}, {"https://s3.us-west-1.amazonaws.com", false}, {"https://s3.dualstack.us-west-1.amazonaws.com", false}, // valid inputs. {"https://bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com", true}, {"https://accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com", true}, {"https://bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com:443", true}, {"https://accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com:443", true}, } for i, testCase := range testCases { u, err := url.Parse(testCase.url) if err != nil { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err) } result := IsAmazonPrivateLinkEndpoint(*u) if testCase.result != result { t.Errorf("Test %d: Expected IsAmazonPrivateLinkEndpoint to be '%v' for input \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.url, result) } } } func TestS3ExpressBucket(t *testing.T) { tests := []struct { bucket string wantErr bool }{ {"my-express-bucket--usw2-az1--x-s3", true}, {"data.analytics--use1-az5--x-s3", true}, {"ml-training--apne1-az4--x-s3", true}, {"my-standard-bucket", false}, {"my-express-bucket--usw2-az1", false}, {"192.168.0.1--usw2-az1--x-s3", false}, {"my..bucket--usw2-az1--x-s3", false}, {"my--bucket--usw2-az1--x-s3", false}, {".mybucket--usw2-az1--x-s3", false}, {"my-bucket--invalid-az7--x-s3", false}, } for _, tt := range tests { t.Run(tt.bucket, func(t *testing.T) { got := IsS3ExpressBucket(tt.bucket) if got != tt.wantErr { t.Errorf("IsS3ExpressBucket(%q) = %v, want %v", tt.bucket, got, tt.wantErr) } }) } } minio-go-7.0.97/pkg/set/000077500000000000000000000000001510244170000147025ustar00rootroot00000000000000minio-go-7.0.97/pkg/set/msgp.go000066400000000000000000000067071510244170000162110ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2025 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package set import "github.com/tinylib/msgp/msgp" // EncodeMsg encodes the message to the writer. // Values are stored as a slice of strings or nil. func (s StringSet) EncodeMsg(writer *msgp.Writer) error { if s == nil { return writer.WriteNil() } err := writer.WriteArrayHeader(uint32(len(s))) if err != nil { return err } sorted := s.ToByteSlices() for _, k := range sorted { err = writer.WriteStringFromBytes(k) if err != nil { return err } } return nil } // MarshalMsg encodes the message to the bytes. // Values are stored as a slice of strings or nil. func (s StringSet) MarshalMsg(bytes []byte) ([]byte, error) { if s == nil { return msgp.AppendNil(bytes), nil } if len(s) == 0 { return msgp.AppendArrayHeader(bytes, 0), nil } bytes = msgp.AppendArrayHeader(bytes, uint32(len(s))) sorted := s.ToByteSlices() for _, k := range sorted { bytes = msgp.AppendStringFromBytes(bytes, k) } return bytes, nil } // DecodeMsg decodes the message from the reader. func (s *StringSet) DecodeMsg(reader *msgp.Reader) error { if reader.IsNil() { *s = nil return reader.Skip() } sz, err := reader.ReadArrayHeader() if err != nil { return err } dst := *s if dst == nil { dst = make(StringSet, sz) } else { for k := range dst { delete(dst, k) } } for i := uint32(0); i < sz; i++ { var k string k, err = reader.ReadString() if err != nil { return err } dst[k] = struct{}{} } *s = dst return nil } // UnmarshalMsg decodes the message from the bytes. func (s *StringSet) UnmarshalMsg(bytes []byte) ([]byte, error) { if msgp.IsNil(bytes) { *s = nil return bytes[msgp.NilSize:], nil } // Read the array header sz, bytes, err := msgp.ReadArrayHeaderBytes(bytes) if err != nil { return nil, err } dst := *s if dst == nil { dst = make(StringSet, sz) } else { for k := range dst { delete(dst, k) } } for i := uint32(0); i < sz; i++ { var k string k, bytes, err = msgp.ReadStringBytes(bytes) if err != nil { return nil, err } dst[k] = struct{}{} } *s = dst return bytes, nil } // Msgsize returns the maximum size of the message. func (s StringSet) Msgsize() int { if s == nil { return msgp.NilSize } if len(s) == 0 { return msgp.ArrayHeaderSize } size := msgp.ArrayHeaderSize for key := range s { size += msgp.StringPrefixSize + len(key) } return size } // MarshalBinary encodes the receiver into a binary form and returns the result. func (s StringSet) MarshalBinary() ([]byte, error) { return s.MarshalMsg(nil) } // AppendBinary appends the binary representation of itself to the end of b func (s StringSet) AppendBinary(b []byte) ([]byte, error) { return s.MarshalMsg(b) } // UnmarshalBinary decodes the binary representation of itself from b func (s *StringSet) UnmarshalBinary(b []byte) error { _, err := s.UnmarshalMsg(b) return err } minio-go-7.0.97/pkg/set/stringset.go000066400000000000000000000124321510244170000172550ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package set import ( "encoding/json" "fmt" "sort" ) // StringSet - uses map as set of strings. type StringSet map[string]struct{} // ToSlice - returns StringSet as string slice. func (set StringSet) ToSlice() []string { keys := make([]string, 0, len(set)) for k := range set { keys = append(keys, k) } sort.Strings(keys) return keys } // ToByteSlices - returns StringSet as a sorted // slice of byte slices, using only one allocation. func (set StringSet) ToByteSlices() [][]byte { length := 0 for k := range set { length += len(k) } // Preallocate the slice with the total length of all strings // to avoid multiple allocations. dst := make([]byte, length) // Add keys to this... keys := make([][]byte, 0, len(set)) for k := range set { n := copy(dst, k) keys = append(keys, dst[:n]) dst = dst[n:] } sort.Slice(keys, func(i, j int) bool { return string(keys[i]) < string(keys[j]) }) return keys } // IsEmpty - returns whether the set is empty or not. func (set StringSet) IsEmpty() bool { return len(set) == 0 } // Add - adds string to the set. func (set StringSet) Add(s string) { set[s] = struct{}{} } // Remove - removes string in the set. It does nothing if string does not exist in the set. func (set StringSet) Remove(s string) { delete(set, s) } // Contains - checks if string is in the set. func (set StringSet) Contains(s string) bool { _, ok := set[s] return ok } // FuncMatch - returns new set containing each value who passes match function. // A 'matchFn' should accept element in a set as first argument and // 'matchString' as second argument. The function can do any logic to // compare both the arguments and should return true to accept element in // a set to include in output set else the element is ignored. func (set StringSet) FuncMatch(matchFn func(string, string) bool, matchString string) StringSet { nset := NewStringSet() for k := range set { if matchFn(k, matchString) { nset.Add(k) } } return nset } // ApplyFunc - returns new set containing each value processed by 'applyFn'. // A 'applyFn' should accept element in a set as a argument and return // a processed string. The function can do any logic to return a processed // string. func (set StringSet) ApplyFunc(applyFn func(string) string) StringSet { nset := NewStringSet() for k := range set { nset.Add(applyFn(k)) } return nset } // Equals - checks whether given set is equal to current set or not. func (set StringSet) Equals(sset StringSet) bool { // If length of set is not equal to length of given set, the // set is not equal to given set. if len(set) != len(sset) { return false } // As both sets are equal in length, check each elements are equal. for k := range set { if _, ok := sset[k]; !ok { return false } } return true } // Intersection - returns the intersection with given set as new set. func (set StringSet) Intersection(sset StringSet) StringSet { nset := NewStringSet() for k := range set { if _, ok := sset[k]; ok { nset.Add(k) } } return nset } // Difference - returns the difference with given set as new set. func (set StringSet) Difference(sset StringSet) StringSet { nset := NewStringSet() for k := range set { if _, ok := sset[k]; !ok { nset.Add(k) } } return nset } // Union - returns the union with given set as new set. func (set StringSet) Union(sset StringSet) StringSet { nset := NewStringSet() for k := range set { nset.Add(k) } for k := range sset { nset.Add(k) } return nset } // MarshalJSON - converts to JSON data. func (set StringSet) MarshalJSON() ([]byte, error) { return json.Marshal(set.ToSlice()) } // UnmarshalJSON - parses JSON data and creates new set with it. func (set *StringSet) UnmarshalJSON(data []byte) error { sl := []interface{}{} var err error if err = json.Unmarshal(data, &sl); err == nil { *set = make(StringSet) for _, s := range sl { set.Add(fmt.Sprintf("%v", s)) } } else { var s interface{} if err = json.Unmarshal(data, &s); err == nil { *set = make(StringSet) set.Add(fmt.Sprintf("%v", s)) } } return err } // String - returns printable string of the set. func (set StringSet) String() string { return fmt.Sprintf("%s", set.ToSlice()) } // NewStringSet - creates new string set. func NewStringSet() StringSet { return make(StringSet) } // CreateStringSet - creates new string set with given string values. func CreateStringSet(sl ...string) StringSet { set := make(StringSet, len(sl)) for _, k := range sl { set.Add(k) } return set } // CopyStringSet - returns copy of given set. func CopyStringSet(set StringSet) StringSet { nset := make(StringSet, len(set)) for k, v := range set { nset[k] = v } return nset } minio-go-7.0.97/pkg/set/stringset_test.go000066400000000000000000000326521510244170000203220ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package set import ( "fmt" "reflect" "sort" "strings" "testing" ) // NewStringSet() is called and the result is validated. func TestNewStringSet(t *testing.T) { if ss := NewStringSet(); !ss.IsEmpty() { t.Fatalf("expected: true, got: false") } } // CreateStringSet() is called and the result is validated. func TestCreateStringSet(t *testing.T) { ss := CreateStringSet("foo") if str := ss.String(); str != `[foo]` { t.Fatalf("expected: %s, got: %s", `["foo"]`, str) } } // CopyStringSet() is called and the result is validated. func TestCopyStringSet(t *testing.T) { ss := CreateStringSet("foo") sscopy := CopyStringSet(ss) if !ss.Equals(sscopy) { t.Fatalf("expected: %s, got: %s", ss, sscopy) } } // StringSet.Add() is called with series of cases for valid and erroneous inputs and the result is validated. func TestStringSetAdd(t *testing.T) { testCases := []struct { value string expectedResult string }{ // Test first addition. {"foo", `[foo]`}, // Test duplicate addition. {"foo", `[foo]`}, // Test new addition. {"bar", `[bar foo]`}, } ss := NewStringSet() for _, testCase := range testCases { ss.Add(testCase.value) if str := ss.String(); str != testCase.expectedResult { t.Fatalf("expected: %s, got: %s", testCase.expectedResult, str) } } } // StringSet.Remove() is called with series of cases for valid and erroneous inputs and the result is validated. func TestStringSetRemove(t *testing.T) { ss := CreateStringSet("foo", "bar") testCases := []struct { value string expectedResult string }{ // Test removing non-existen item. {"baz", `[bar foo]`}, // Test remove existing item. {"foo", `[bar]`}, // Test remove existing item again. {"foo", `[bar]`}, // Test remove to make set to empty. {"bar", `[]`}, } for _, testCase := range testCases { ss.Remove(testCase.value) if str := ss.String(); str != testCase.expectedResult { t.Fatalf("expected: %s, got: %s", testCase.expectedResult, str) } } } // StringSet.Contains() is called with series of cases for valid and erroneous inputs and the result is validated. func TestStringSetContains(t *testing.T) { ss := CreateStringSet("foo") testCases := []struct { value string expectedResult bool }{ // Test to check non-existent item. {"bar", false}, // Test to check existent item. {"foo", true}, // Test to verify case sensitivity. {"Foo", false}, } for _, testCase := range testCases { if result := ss.Contains(testCase.value); result != testCase.expectedResult { t.Fatalf("expected: %t, got: %t", testCase.expectedResult, result) } } } // StringSet.FuncMatch() is called with series of cases for valid and erroneous inputs and the result is validated. func TestStringSetFuncMatch(t *testing.T) { ss := CreateStringSet("foo", "bar") testCases := []struct { matchFn func(string, string) bool value string expectedResult string }{ // Test to check match function doing case insensive compare. {func(setValue, compareValue string) bool { return strings.EqualFold(setValue, compareValue) }, "Bar", `[bar]`}, // Test to check match function doing prefix check. {func(setValue, compareValue string) bool { return strings.HasPrefix(compareValue, setValue) }, "foobar", `[foo]`}, } for _, testCase := range testCases { s := ss.FuncMatch(testCase.matchFn, testCase.value) if result := s.String(); result != testCase.expectedResult { t.Fatalf("expected: %s, got: %s", testCase.expectedResult, result) } } } // StringSet.ApplyFunc() is called with series of cases for valid and erroneous inputs and the result is validated. func TestStringSetApplyFunc(t *testing.T) { ss := CreateStringSet("foo", "bar") testCases := []struct { applyFn func(string) string expectedResult string }{ // Test to apply function prepending a known string. {func(setValue string) string { return "mybucket/" + setValue }, `[mybucket/bar mybucket/foo]`}, // Test to apply function modifying values. {func(setValue string) string { return setValue[1:] }, `[ar oo]`}, } for _, testCase := range testCases { s := ss.ApplyFunc(testCase.applyFn) if result := s.String(); result != testCase.expectedResult { t.Fatalf("expected: %s, got: %s", testCase.expectedResult, result) } } } // StringSet.Equals() is called with series of cases for valid and erroneous inputs and the result is validated. func TestStringSetEquals(t *testing.T) { testCases := []struct { set1 StringSet set2 StringSet expectedResult bool }{ // Test equal set {CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar"), true}, // Test second set with more items {CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar", "baz"), false}, // Test second set with less items {CreateStringSet("foo", "bar"), CreateStringSet("bar"), false}, } for _, testCase := range testCases { if result := testCase.set1.Equals(testCase.set2); result != testCase.expectedResult { t.Fatalf("expected: %t, got: %t", testCase.expectedResult, result) } } } // StringSet.Intersection() is called with series of cases for valid and erroneous inputs and the result is validated. func TestStringSetIntersection(t *testing.T) { testCases := []struct { set1 StringSet set2 StringSet expectedResult StringSet }{ // Test intersecting all values. {CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar")}, // Test intersecting all values in second set. {CreateStringSet("foo", "bar", "baz"), CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar")}, // Test intersecting different values in second set. {CreateStringSet("foo", "baz"), CreateStringSet("baz", "bar"), CreateStringSet("baz")}, // Test intersecting none. {CreateStringSet("foo", "baz"), CreateStringSet("poo", "bar"), NewStringSet()}, } for _, testCase := range testCases { if result := testCase.set1.Intersection(testCase.set2); !result.Equals(testCase.expectedResult) { t.Fatalf("expected: %s, got: %s", testCase.expectedResult, result) } } } // StringSet.Difference() is called with series of cases for valid and erroneous inputs and the result is validated. func TestStringSetDifference(t *testing.T) { testCases := []struct { set1 StringSet set2 StringSet expectedResult StringSet }{ // Test differing none. {CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar"), NewStringSet()}, // Test differing in first set. {CreateStringSet("foo", "bar", "baz"), CreateStringSet("foo", "bar"), CreateStringSet("baz")}, // Test differing values in both set. {CreateStringSet("foo", "baz"), CreateStringSet("baz", "bar"), CreateStringSet("foo")}, // Test differing all values. {CreateStringSet("foo", "baz"), CreateStringSet("poo", "bar"), CreateStringSet("foo", "baz")}, } for _, testCase := range testCases { if result := testCase.set1.Difference(testCase.set2); !result.Equals(testCase.expectedResult) { t.Fatalf("expected: %s, got: %s", testCase.expectedResult, result) } } } // StringSet.Union() is called with series of cases for valid and erroneous inputs and the result is validated. func TestStringSetUnion(t *testing.T) { testCases := []struct { set1 StringSet set2 StringSet expectedResult StringSet }{ // Test union same values. {CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar")}, // Test union same values in second set. {CreateStringSet("foo", "bar", "baz"), CreateStringSet("foo", "bar"), CreateStringSet("foo", "bar", "baz")}, // Test union different values in both set. {CreateStringSet("foo", "baz"), CreateStringSet("baz", "bar"), CreateStringSet("foo", "baz", "bar")}, // Test union all different values. {CreateStringSet("foo", "baz"), CreateStringSet("poo", "bar"), CreateStringSet("foo", "baz", "poo", "bar")}, } for _, testCase := range testCases { if result := testCase.set1.Union(testCase.set2); !result.Equals(testCase.expectedResult) { t.Fatalf("expected: %s, got: %s", testCase.expectedResult, result) } } } // StringSet.MarshalJSON() is called with series of cases for valid and erroneous inputs and the result is validated. func TestStringSetMarshalJSON(t *testing.T) { testCases := []struct { set StringSet expectedResult string }{ // Test set with values. {CreateStringSet("foo", "bar"), `["bar","foo"]`}, // Test empty set. {NewStringSet(), "[]"}, } for _, testCase := range testCases { if result, _ := testCase.set.MarshalJSON(); string(result) != testCase.expectedResult { t.Fatalf("expected: %s, got: %s", testCase.expectedResult, string(result)) } } } // StringSet.UnmarshalJSON() is called with series of cases for valid and erroneous inputs and the result is validated. func TestStringSetUnmarshalJSON(t *testing.T) { testCases := []struct { data []byte expectedResult string }{ // Test to convert JSON array to set. {[]byte(`["bar","foo"]`), `[bar foo]`}, // Test to convert JSON string to set. {[]byte(`"bar"`), `[bar]`}, // Test to convert JSON empty array to set. {[]byte(`[]`), `[]`}, // Test to convert JSON empty string to set. {[]byte(`""`), `[]`}, } for _, testCase := range testCases { var set StringSet set.UnmarshalJSON(testCase.data) if result := set.String(); result != testCase.expectedResult { t.Fatalf("expected: %s, got: %s", testCase.expectedResult, result) } } } // StringSet.String() is called with series of cases for valid and erroneous inputs and the result is validated. func TestStringSetString(t *testing.T) { testCases := []struct { set StringSet expectedResult string }{ // Test empty set. {NewStringSet(), `[]`}, // Test set with empty value. {CreateStringSet(""), `[]`}, // Test set with value. {CreateStringSet("foo"), `[foo]`}, } for _, testCase := range testCases { if str := testCase.set.String(); str != testCase.expectedResult { t.Fatalf("expected: %s, got: %s", testCase.expectedResult, str) } } } // StringSet.ToSlice() is called with series of cases for valid and erroneous inputs and the result is validated. func TestStringSetToSlice(t *testing.T) { testCases := []struct { set StringSet expectedResult string }{ // Test empty set. {NewStringSet(), `[]`}, // Test set with empty value. {CreateStringSet(""), `[]`}, // Test set with value. {CreateStringSet("foo"), `[foo]`}, // Test set with value. {CreateStringSet("foo", "bar"), `[bar foo]`}, } for _, testCase := range testCases { sslice := testCase.set.ToSlice() if str := fmt.Sprintf("%s", sslice); str != testCase.expectedResult { t.Fatalf("expected: %s, got: %s", testCase.expectedResult, str) } } } func TestStringSet_UnmarshalJSON(t *testing.T) { type args struct { data []byte expectResult []string } tests := []struct { name string set StringSet args args wantErr bool }{ { name: "test strings", set: NewStringSet(), args: args{ data: []byte(`["foo","bar"]`), expectResult: []string{"foo", "bar"}, }, wantErr: false, }, { name: "test string", set: NewStringSet(), args: args{ data: []byte(`"foo"`), expectResult: []string{"foo"}, }, wantErr: false, }, { name: "test bools", set: NewStringSet(), args: args{ data: []byte(`[false,true]`), expectResult: []string{"false", "true"}, }, wantErr: false, }, { name: "test bool", set: NewStringSet(), args: args{ data: []byte(`false`), expectResult: []string{"false"}, }, wantErr: false, }, { name: "test ints", set: NewStringSet(), args: args{ data: []byte(`[1,2]`), expectResult: []string{"1", "2"}, }, wantErr: false, }, { name: "test int", set: NewStringSet(), args: args{ data: []byte(`1`), expectResult: []string{"1"}, }, wantErr: false, }, { name: "test floats", set: NewStringSet(), args: args{ data: []byte(`[1.1,2.2]`), expectResult: []string{"1.1", "2.2"}, }, wantErr: false, }, { name: "test float", set: NewStringSet(), args: args{ data: []byte(`1.1`), expectResult: []string{"1.1"}, }, wantErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if err := tt.set.UnmarshalJSON(tt.args.data); (err != nil) != tt.wantErr { t.Errorf("UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr) } slice := tt.set.ToSlice() sort.Slice(slice, func(i, j int) bool { return slice[i] < slice[j] }) sort.Slice(tt.args.expectResult, func(i, j int) bool { return tt.args.expectResult[i] < tt.args.expectResult[j] }) if !reflect.DeepEqual(slice, tt.args.expectResult) { t.Errorf("StringSet() get %v, want %v", tt.set.ToSlice(), tt.args.expectResult) } }) } } minio-go-7.0.97/pkg/signer/000077500000000000000000000000001510244170000153765ustar00rootroot00000000000000minio-go-7.0.97/pkg/signer/request-signature-streaming-unsigned-trailer.go000066400000000000000000000152241510244170000266610ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2022 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package signer import ( "bytes" "fmt" "io" "net/http" "strconv" "strings" "time" ) // getUnsignedChunkLength - calculates the length of chunk metadata func getUnsignedChunkLength(chunkDataSize int64) int64 { return int64(len(fmt.Sprintf("%x", chunkDataSize))) + crlfLen + chunkDataSize + crlfLen } // getUSStreamLength - calculates the length of the overall stream (data + metadata) func getUSStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 { if dataLen <= 0 { return 0 } chunksCount := int64(dataLen / chunkSize) remainingBytes := int64(dataLen % chunkSize) streamLen := int64(0) streamLen += chunksCount * getUnsignedChunkLength(chunkSize) if remainingBytes > 0 { streamLen += getUnsignedChunkLength(remainingBytes) } streamLen += getUnsignedChunkLength(0) if len(trailers) > 0 { for name, placeholder := range trailers { if len(placeholder) > 0 { streamLen += int64(len(name) + len(trailerKVSeparator) + len(placeholder[0]) + 1) } } streamLen += crlfLen } return streamLen } // prepareStreamingRequest - prepares a request with appropriate // headers before computing the seed signature. func prepareUSStreamingRequest(req *http.Request, sessionToken string, dataLen int64, timestamp time.Time) { req.TransferEncoding = []string{"aws-chunked"} if sessionToken != "" { req.Header.Set("X-Amz-Security-Token", sessionToken) } req.Header.Set("X-Amz-Date", timestamp.Format(iso8601DateFormat)) // Set content length with streaming signature for each chunk included. req.ContentLength = getUSStreamLength(dataLen, int64(payloadChunkSize), req.Trailer) } // StreamingUSReader implements chunked upload signature as a reader on // top of req.Body's ReaderCloser chunk header;data;... repeat type StreamingUSReader struct { contentLen int64 // Content-Length from req header baseReadCloser io.ReadCloser // underlying io.Reader bytesRead int64 // bytes read from underlying io.Reader buf bytes.Buffer // holds signed chunk chunkBuf []byte // holds raw data read from req Body chunkBufLen int // no. of bytes read so far into chunkBuf done bool // done reading the underlying reader to EOF chunkNum int totalChunks int lastChunkSize int trailer http.Header } // writeChunk - signs a chunk read from s.baseReader of chunkLen size. func (s *StreamingUSReader) writeChunk(chunkLen int, addCrLf bool) { s.buf.WriteString(strconv.FormatInt(int64(chunkLen), 16) + "\r\n") // Write chunk data into streaming buffer s.buf.Write(s.chunkBuf[:chunkLen]) // Write the chunk trailer. if addCrLf { s.buf.Write([]byte("\r\n")) } // Reset chunkBufLen for next chunk read. s.chunkBufLen = 0 s.chunkNum++ } // addSignedTrailer - adds a trailer with the provided headers, // then signs a chunk and adds it to output. func (s *StreamingUSReader) addTrailer(h http.Header) { olen := len(s.chunkBuf) s.chunkBuf = s.chunkBuf[:0] for k, v := range h { s.chunkBuf = append(s.chunkBuf, []byte(strings.ToLower(k)+trailerKVSeparator+v[0]+"\n")...) } s.buf.Write(s.chunkBuf) s.buf.WriteString("\r\n\r\n") // Reset chunkBufLen for next chunk read. s.chunkBuf = s.chunkBuf[:olen] s.chunkBufLen = 0 s.chunkNum++ } // StreamingUnsignedV4 - provides chunked upload func StreamingUnsignedV4(req *http.Request, sessionToken string, dataLen int64, reqTime time.Time) *http.Request { // Set headers needed for streaming signature. prepareUSStreamingRequest(req, sessionToken, dataLen, reqTime) if req.Body == nil { req.Body = io.NopCloser(bytes.NewReader([]byte(""))) } stReader := &StreamingUSReader{ baseReadCloser: req.Body, chunkBuf: make([]byte, payloadChunkSize), contentLen: dataLen, chunkNum: 1, totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1, lastChunkSize: int(dataLen % payloadChunkSize), } if len(req.Trailer) > 0 { stReader.trailer = req.Trailer // Remove... req.Trailer = nil } req.Body = stReader return req } // Read - this method performs chunk upload signature providing a // io.Reader interface. func (s *StreamingUSReader) Read(buf []byte) (int, error) { switch { // After the last chunk is read from underlying reader, we // never re-fill s.buf. case s.done: // s.buf will be (re-)filled with next chunk when has lesser // bytes than asked for. case s.buf.Len() < len(buf): s.chunkBufLen = 0 for { n1, err := s.baseReadCloser.Read(s.chunkBuf[s.chunkBufLen:]) // Usually we validate `err` first, but in this case // we are validating n > 0 for the following reasons. // // 1. n > 0, err is one of io.EOF, nil (near end of stream) // A Reader returning a non-zero number of bytes at the end // of the input stream may return either err == EOF or err == nil // // 2. n == 0, err is io.EOF (actual end of stream) // // Callers should always process the n > 0 bytes returned // before considering the error err. if n1 > 0 { s.chunkBufLen += n1 s.bytesRead += int64(n1) if s.chunkBufLen == payloadChunkSize || (s.chunkNum == s.totalChunks-1 && s.chunkBufLen == s.lastChunkSize) { // Sign the chunk and write it to s.buf. s.writeChunk(s.chunkBufLen, true) break } } if err != nil { if err == io.EOF { // No more data left in baseReader - last chunk. // Done reading the last chunk from baseReader. s.done = true // bytes read from baseReader different than // content length provided. if s.bytesRead != s.contentLen { return 0, fmt.Errorf("http: ContentLength=%d with Body length %d", s.contentLen, s.bytesRead) } // Sign the chunk and write it to s.buf. s.writeChunk(0, len(s.trailer) == 0) if len(s.trailer) > 0 { // Trailer must be set now. s.addTrailer(s.trailer) } break } return 0, err } } } return s.buf.Read(buf) } // Close - this method makes underlying io.ReadCloser's Close method available. func (s *StreamingUSReader) Close() error { return s.baseReadCloser.Close() } minio-go-7.0.97/pkg/signer/request-signature-streaming.go000066400000000000000000000347031510244170000234120ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package signer import ( "bytes" "encoding/hex" "fmt" "io" "net/http" "strconv" "strings" "time" md5simd "github.com/minio/md5-simd" ) // Reference for constants used below - // http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming const ( streamingSignAlgorithm = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" streamingSignTrailerAlgorithm = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" streamingPayloadHdr = "AWS4-HMAC-SHA256-PAYLOAD" streamingTrailerHdr = "AWS4-HMAC-SHA256-TRAILER" emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" payloadChunkSize = 64 * 1024 chunkSigConstLen = 17 // ";chunk-signature=" signatureStrLen = 64 // e.g. "f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2" crlfLen = 2 // CRLF trailerKVSeparator = ":" trailerSignature = "x-amz-trailer-signature" ) // Request headers to be ignored while calculating seed signature for // a request. var ignoredStreamingHeaders = map[string]bool{ "Authorization": true, "User-Agent": true, "Content-Type": true, } // getSignedChunkLength - calculates the length of chunk metadata func getSignedChunkLength(chunkDataSize int64) int64 { return int64(len(fmt.Sprintf("%x", chunkDataSize))) + chunkSigConstLen + signatureStrLen + crlfLen + chunkDataSize + crlfLen } // getStreamLength - calculates the length of the overall stream (data + metadata) func getStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 { if dataLen <= 0 { return 0 } chunksCount := int64(dataLen / chunkSize) remainingBytes := int64(dataLen % chunkSize) streamLen := int64(0) streamLen += chunksCount * getSignedChunkLength(chunkSize) if remainingBytes > 0 { streamLen += getSignedChunkLength(remainingBytes) } streamLen += getSignedChunkLength(0) if len(trailers) > 0 { for name, placeholder := range trailers { if len(placeholder) > 0 { streamLen += int64(len(name) + len(trailerKVSeparator) + len(placeholder[0]) + 1) } } streamLen += int64(len(trailerSignature)+len(trailerKVSeparator)) + signatureStrLen + crlfLen + crlfLen } return streamLen } // buildChunkStringToSign - returns the string to sign given chunk data // and previous signature. func buildChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string { stringToSignParts := []string{ streamingPayloadHdr, t.Format(iso8601DateFormat), getScope(region, t, ServiceTypeS3), previousSig, emptySHA256, chunkChecksum, } return strings.Join(stringToSignParts, "\n") } // buildTrailerChunkStringToSign - returns the string to sign given chunk data // and previous signature. func buildTrailerChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string { stringToSignParts := []string{ streamingTrailerHdr, t.Format(iso8601DateFormat), getScope(region, t, ServiceTypeS3), previousSig, chunkChecksum, } return strings.Join(stringToSignParts, "\n") } // prepareStreamingRequest - prepares a request with appropriate // headers before computing the seed signature. func prepareStreamingRequest(req *http.Request, sessionToken string, dataLen int64, timestamp time.Time) { // Set x-amz-content-sha256 header. if len(req.Trailer) == 0 { req.Header.Set("X-Amz-Content-Sha256", streamingSignAlgorithm) } else { req.Header.Set("X-Amz-Content-Sha256", streamingSignTrailerAlgorithm) for k := range req.Trailer { req.Header.Add("X-Amz-Trailer", strings.ToLower(k)) } req.TransferEncoding = []string{"aws-chunked"} } if sessionToken != "" { req.Header.Set("X-Amz-Security-Token", sessionToken) } req.Header.Set("X-Amz-Date", timestamp.Format(iso8601DateFormat)) // Set content length with streaming signature for each chunk included. req.ContentLength = getStreamLength(dataLen, int64(payloadChunkSize), req.Trailer) req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(dataLen, 10)) } // buildChunkHeader - returns the chunk header. // e.g string(IntHexBase(chunk-size)) + ";chunk-signature=" + signature + \r\n + chunk-data + \r\n func buildChunkHeader(chunkLen int64, signature string) []byte { return []byte(strconv.FormatInt(chunkLen, 16) + ";chunk-signature=" + signature + "\r\n") } // buildChunkSignature - returns chunk signature for a given chunk and previous signature. func buildChunkSignature(chunkCheckSum string, reqTime time.Time, region, previousSignature, secretAccessKey string, ) string { chunkStringToSign := buildChunkStringToSign(reqTime, region, previousSignature, chunkCheckSum) signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3) return getSignature(signingKey, chunkStringToSign) } // buildChunkSignature - returns chunk signature for a given chunk and previous signature. func buildTrailerChunkSignature(chunkChecksum string, reqTime time.Time, region, previousSignature, secretAccessKey string, ) string { chunkStringToSign := buildTrailerChunkStringToSign(reqTime, region, previousSignature, chunkChecksum) signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3) return getSignature(signingKey, chunkStringToSign) } // getSeedSignature - returns the seed signature for a given request. func (s *StreamingReader) setSeedSignature(req *http.Request) { // Get canonical request canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders, getHashedPayload(*req)) // Get string to sign from canonical request. stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest, ServiceTypeS3) signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime, ServiceTypeS3) // Calculate signature. s.seedSignature = getSignature(signingKey, stringToSign) } // StreamingReader implements chunked upload signature as a reader on // top of req.Body's ReaderCloser chunk header;data;... repeat type StreamingReader struct { accessKeyID string secretAccessKey string sessionToken string region string prevSignature string seedSignature string contentLen int64 // Content-Length from req header baseReadCloser io.ReadCloser // underlying io.Reader bytesRead int64 // bytes read from underlying io.Reader buf bytes.Buffer // holds signed chunk chunkBuf []byte // holds raw data read from req Body chunkBufLen int // no. of bytes read so far into chunkBuf done bool // done reading the underlying reader to EOF reqTime time.Time chunkNum int totalChunks int lastChunkSize int trailer http.Header sh256 md5simd.Hasher } // signChunk - signs a chunk read from s.baseReader of chunkLen size. func (s *StreamingReader) signChunk(chunkLen int, addCrLf bool) { // Compute chunk signature for next header s.sh256.Reset() s.sh256.Write(s.chunkBuf[:chunkLen]) chunckChecksum := hex.EncodeToString(s.sh256.Sum(nil)) signature := buildChunkSignature(chunckChecksum, s.reqTime, s.region, s.prevSignature, s.secretAccessKey) // For next chunk signature computation s.prevSignature = signature // Write chunk header into streaming buffer chunkHdr := buildChunkHeader(int64(chunkLen), signature) s.buf.Write(chunkHdr) // Write chunk data into streaming buffer s.buf.Write(s.chunkBuf[:chunkLen]) // Write the chunk trailer. if addCrLf { s.buf.Write([]byte("\r\n")) } // Reset chunkBufLen for next chunk read. s.chunkBufLen = 0 s.chunkNum++ } // addSignedTrailer - adds a trailer with the provided headers, // then signs a chunk and adds it to output. func (s *StreamingReader) addSignedTrailer(h http.Header) { olen := len(s.chunkBuf) s.chunkBuf = s.chunkBuf[:0] for k, v := range h { s.chunkBuf = append(s.chunkBuf, []byte(strings.ToLower(k)+trailerKVSeparator+v[0]+"\n")...) } s.sh256.Reset() s.sh256.Write(s.chunkBuf) chunkChecksum := hex.EncodeToString(s.sh256.Sum(nil)) // Compute chunk signature signature := buildTrailerChunkSignature(chunkChecksum, s.reqTime, s.region, s.prevSignature, s.secretAccessKey) // For next chunk signature computation s.prevSignature = signature s.buf.Write(s.chunkBuf) s.buf.WriteString("\r\n" + trailerSignature + trailerKVSeparator + signature + "\r\n\r\n") // Reset chunkBufLen for next chunk read. s.chunkBuf = s.chunkBuf[:olen] s.chunkBufLen = 0 s.chunkNum++ } // setStreamingAuthHeader - builds and sets authorization header value // for streaming signature. func (s *StreamingReader) setStreamingAuthHeader(req *http.Request, serviceType string) { credential := GetCredential(s.accessKeyID, s.region, s.reqTime, serviceType) authParts := []string{ signV4Algorithm + " Credential=" + credential, "SignedHeaders=" + getSignedHeaders(*req, ignoredStreamingHeaders), "Signature=" + s.seedSignature, } // Set authorization header. auth := strings.Join(authParts, ",") req.Header.Set("Authorization", auth) } // StreamingSignV4Express - provides chunked upload signatureV4 support by // implementing io.Reader. func StreamingSignV4Express(req *http.Request, accessKeyID, secretAccessKey, sessionToken, region string, dataLen int64, reqTime time.Time, sh256 md5simd.Hasher, ) *http.Request { // Set headers needed for streaming signature. prepareStreamingRequest(req, sessionToken, dataLen, reqTime) if req.Body == nil { req.Body = io.NopCloser(bytes.NewReader([]byte(""))) } stReader := &StreamingReader{ baseReadCloser: req.Body, accessKeyID: accessKeyID, secretAccessKey: secretAccessKey, sessionToken: sessionToken, region: region, reqTime: reqTime, chunkBuf: make([]byte, payloadChunkSize), contentLen: dataLen, chunkNum: 1, totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1, lastChunkSize: int(dataLen % payloadChunkSize), sh256: sh256, } if len(req.Trailer) > 0 { stReader.trailer = req.Trailer // Remove... req.Trailer = nil } // Add the request headers required for chunk upload signing. // Compute the seed signature. stReader.setSeedSignature(req) // Set the authorization header with the seed signature. stReader.setStreamingAuthHeader(req, ServiceTypeS3Express) // Set seed signature as prevSignature for subsequent // streaming signing process. stReader.prevSignature = stReader.seedSignature req.Body = stReader return req } // StreamingSignV4 - provides chunked upload signatureV4 support by // implementing io.Reader. func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionToken, region string, dataLen int64, reqTime time.Time, sh256 md5simd.Hasher, ) *http.Request { // Set headers needed for streaming signature. prepareStreamingRequest(req, sessionToken, dataLen, reqTime) if req.Body == nil { req.Body = io.NopCloser(bytes.NewReader([]byte(""))) } stReader := &StreamingReader{ baseReadCloser: req.Body, accessKeyID: accessKeyID, secretAccessKey: secretAccessKey, sessionToken: sessionToken, region: region, reqTime: reqTime, chunkBuf: make([]byte, payloadChunkSize), contentLen: dataLen, chunkNum: 1, totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1, lastChunkSize: int(dataLen % payloadChunkSize), sh256: sh256, } if len(req.Trailer) > 0 { stReader.trailer = req.Trailer // Remove... req.Trailer = nil } // Add the request headers required for chunk upload signing. // Compute the seed signature. stReader.setSeedSignature(req) // Set the authorization header with the seed signature. stReader.setStreamingAuthHeader(req, ServiceTypeS3) // Set seed signature as prevSignature for subsequent // streaming signing process. stReader.prevSignature = stReader.seedSignature req.Body = stReader return req } // Read - this method performs chunk upload signature providing a // io.Reader interface. func (s *StreamingReader) Read(buf []byte) (int, error) { switch { // After the last chunk is read from underlying reader, we // never re-fill s.buf. case s.done: // s.buf will be (re-)filled with next chunk when has lesser // bytes than asked for. case s.buf.Len() < len(buf): s.chunkBufLen = 0 for { n1, err := s.baseReadCloser.Read(s.chunkBuf[s.chunkBufLen:]) // Usually we validate `err` first, but in this case // we are validating n > 0 for the following reasons. // // 1. n > 0, err is one of io.EOF, nil (near end of stream) // A Reader returning a non-zero number of bytes at the end // of the input stream may return either err == EOF or err == nil // // 2. n == 0, err is io.EOF (actual end of stream) // // Callers should always process the n > 0 bytes returned // before considering the error err. if n1 > 0 { s.chunkBufLen += n1 s.bytesRead += int64(n1) if s.chunkBufLen == payloadChunkSize || (s.chunkNum == s.totalChunks-1 && s.chunkBufLen == s.lastChunkSize) { // Sign the chunk and write it to s.buf. s.signChunk(s.chunkBufLen, true) break } } if err != nil { if err == io.EOF { // No more data left in baseReader - last chunk. // Done reading the last chunk from baseReader. s.done = true // bytes read from baseReader different than // content length provided. if s.bytesRead != s.contentLen { return 0, fmt.Errorf("http: ContentLength=%d with Body length %d", s.contentLen, s.bytesRead) } // Sign the chunk and write it to s.buf. s.signChunk(0, len(s.trailer) == 0) if len(s.trailer) > 0 { // Trailer must be set now. s.addSignedTrailer(s.trailer) } break } return 0, err } } } return s.buf.Read(buf) } // Close - this method makes underlying io.ReadCloser's Close method available. func (s *StreamingReader) Close() error { if s.sh256 != nil { s.sh256.Close() s.sh256 = nil } return s.baseReadCloser.Close() } minio-go-7.0.97/pkg/signer/request-signature-streaming_test.go000066400000000000000000000156501510244170000244510ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package signer import ( "bytes" "crypto/sha256" "encoding/hex" "hash" "io" "net/http" "testing" "time" md5simd "github.com/minio/md5-simd" ) // hashWrapper implements the md5simd.Hasher interface. type hashWrapper struct { hash.Hash } func newSHA256Hasher() md5simd.Hasher { return &hashWrapper{Hash: sha256.New()} } func (m *hashWrapper) Close() { m.Hash = nil } func sum256hex(data []byte) string { hash := sha256.New() hash.Write(data) return hex.EncodeToString(hash.Sum(nil)) } func TestGetSeedSignature(t *testing.T) { accessKeyID := "AKIAIOSFODNN7EXAMPLE" secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" dataLen := 66560 data := bytes.Repeat([]byte("a"), dataLen) body := io.NopCloser(bytes.NewReader(data)) req := NewRequest(http.MethodPut, "/examplebucket/chunkObject.txt", body) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") req.Host = "s3.amazonaws.com" reqTime, err := time.Parse("20060102T150405Z", "20130524T000000Z") if err != nil { t.Fatalf("Failed to parse time - %v", err) } req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", "us-east-1", int64(dataLen), reqTime, newSHA256Hasher()) actualSeedSignature := req.Body.(*StreamingReader).seedSignature expectedSeedSignature := "38cab3af09aa15ddf29e26e36236f60fb6bfb6243a20797ae9a8183674526079" if actualSeedSignature != expectedSeedSignature { t.Errorf("Expected %s but received %s", expectedSeedSignature, actualSeedSignature) } } func TestChunkSignature(t *testing.T) { chunkData := bytes.Repeat([]byte("a"), 65536) reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z") previousSignature := "4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9" location := "us-east-1" secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" expectedSignature := "ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648" chunkCheckSum := sum256hex(chunkData) actualSignature := buildChunkSignature(chunkCheckSum, reqTime, location, previousSignature, secretAccessKeyID) if actualSignature != expectedSignature { t.Errorf("Expected %s but received %s", expectedSignature, actualSignature) } } // Example on https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html func TestTrailerChunkSignature(t *testing.T) { chunkData := []byte("x-amz-checksum-crc32c:wdBDMA==\n") reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z") previousSignature := "e05ab64fe1dfdbf0b5870abbaabdb063c371d4e96f2767e6934d90529c5ae850" location := "us-east-1" secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" expectedSignature := "41e14ac611e27a8bb3d66c3bad6856f209297767d5dd4fc87d8fa9e422e03faf" chunkCheckSum := sum256hex(chunkData) actualSignature := buildTrailerChunkSignature(chunkCheckSum, reqTime, location, previousSignature, secretAccessKeyID) if actualSignature != expectedSignature { t.Errorf("Expected %s but received %s", expectedSignature, actualSignature) } } func TestSetStreamingAuthorization(t *testing.T) { location := "us-east-1" secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" accessKeyID := "AKIAIOSFODNN7EXAMPLE" req := NewRequest(http.MethodPut, "/examplebucket/chunkObject.txt", nil) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") req.Host = "" req.URL.Host = "s3.amazonaws.com" dataLen := int64(65 * 1024) reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z") req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime, newSHA256Hasher()) expectedAuthorization := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=38cab3af09aa15ddf29e26e36236f60fb6bfb6243a20797ae9a8183674526079" actualAuthorization := req.Header.Get("Authorization") if actualAuthorization != expectedAuthorization { t.Errorf("Expected %s but received %s", expectedAuthorization, actualAuthorization) } } // Test against https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html func TestSetStreamingAuthorizationTrailer(t *testing.T) { location := "us-east-1" secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" accessKeyID := "AKIAIOSFODNN7EXAMPLE" req := NewRequest(http.MethodPut, "/examplebucket/chunkObject.txt", nil) req.Header.Set("Content-Encoding", "aws-chunked") req.Header.Set("x-amz-decoded-content-length", "66560") req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") req.Host = "" req.URL.Host = "s3.amazonaws.com" req.Trailer = http.Header{} req.Trailer.Set("x-amz-checksum-crc32c", "wdBDMA==") dataLen := int64(65 * 1024) reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z") req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime, newSHA256Hasher()) // (order of signed headers is different) expectedAuthorization := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class;x-amz-trailer,Signature=106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e" actualAuthorization := req.Header.Get("Authorization") if actualAuthorization != expectedAuthorization { t.Errorf("Expected \n%s but received \n%s", expectedAuthorization, actualAuthorization) } chunkData := []byte("x-amz-checksum-crc32c:wdBDMA==\n") t.Log(hex.EncodeToString(sum256(chunkData))) } func TestStreamingReader(t *testing.T) { reqTime, _ := time.Parse("20060102T150405Z", "20130524T000000Z") location := "us-east-1" secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" accessKeyID := "AKIAIOSFODNN7EXAMPLE" dataLen := int64(65 * 1024) req := NewRequest(http.MethodPut, "/examplebucket/chunkObject.txt", nil) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") req.ContentLength = 65 * 1024 req.Host = "" req.URL.Host = "s3.amazonaws.com" baseReader := io.NopCloser(bytes.NewReader(bytes.Repeat([]byte("a"), 65*1024))) req.Body = baseReader req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime, newSHA256Hasher()) b, err := io.ReadAll(req.Body) if err != nil { t.Errorf("Expected no error but received %v %d", err, len(b)) } req.Body.Close() } minio-go-7.0.97/pkg/signer/request-signature-v2.go000066400000000000000000000215071510244170000217460ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package signer import ( "bytes" "crypto/hmac" "crypto/sha1" "encoding/base64" "fmt" "net/http" "net/url" "sort" "strconv" "strings" "time" "github.com/minio/minio-go/v7/pkg/s3utils" ) // Signature and API related constants. const ( signV2Algorithm = "AWS" ) // Encode input URL path to URL encoded path. func encodeURL2Path(req *http.Request, virtualHost bool) (path string) { if virtualHost { reqHost := getHostAddr(req) dotPos := strings.Index(reqHost, ".") if dotPos > -1 { bucketName := reqHost[:dotPos] path = "/" + bucketName path += req.URL.Path path = s3utils.EncodePath(path) return path } } path = s3utils.EncodePath(req.URL.Path) return path } // PreSignV2 - presign the request in following style. // https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE}. func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires int64, virtualHost bool) *http.Request { // Presign is not needed for anonymous credentials. if accessKeyID == "" || secretAccessKey == "" { return &req } d := time.Now().UTC() // Find epoch expires when the request will expire. epochExpires := d.Unix() + expires // Add expires header if not present. if expiresStr := req.Header.Get("Expires"); expiresStr == "" { req.Header.Set("Expires", strconv.FormatInt(epochExpires, 10)) } // Get presigned string to sign. stringToSign := preStringToSignV2(req, virtualHost) hm := hmac.New(sha1.New, []byte(secretAccessKey)) hm.Write([]byte(stringToSign)) // Calculate signature. signature := base64.StdEncoding.EncodeToString(hm.Sum(nil)) query := req.URL.Query() // Handle specially for Google Cloud Storage. if strings.Contains(getHostAddr(&req), ".storage.googleapis.com") { query.Set("GoogleAccessId", accessKeyID) } else { query.Set("AWSAccessKeyId", accessKeyID) } // Fill in Expires for presigned query. query.Set("Expires", strconv.FormatInt(epochExpires, 10)) // Encode query and save. req.URL.RawQuery = s3utils.QueryEncode(query) // Save signature finally. req.URL.RawQuery += "&Signature=" + s3utils.EncodePath(signature) // Return. return &req } // PostPresignSignatureV2 - presigned signature for PostPolicy // request. func PostPresignSignatureV2(policyBase64, secretAccessKey string) string { hm := hmac.New(sha1.New, []byte(secretAccessKey)) hm.Write([]byte(policyBase64)) signature := base64.StdEncoding.EncodeToString(hm.Sum(nil)) return signature } // Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature; // Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID, UTF-8-Encoding-Of( StringToSign ) ) ); // // StringToSign = HTTP-Verb + "\n" + // Content-Md5 + "\n" + // Content-Type + "\n" + // Date + "\n" + // CanonicalizedProtocolHeaders + // CanonicalizedResource; // // CanonicalizedResource = [ "/" + Bucket ] + // + // [ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"]; // // CanonicalizedProtocolHeaders = // SignV2 sign the request before Do() (AWS Signature Version 2). func SignV2(req http.Request, accessKeyID, secretAccessKey string, virtualHost bool) *http.Request { // Signature calculation is not needed for anonymous credentials. if accessKeyID == "" || secretAccessKey == "" { return &req } // Initial time. d := time.Now().UTC() // Add date if not present. if date := req.Header.Get("Date"); date == "" { req.Header.Set("Date", d.Format(http.TimeFormat)) } // Calculate HMAC for secretAccessKey. stringToSign := stringToSignV2(req, virtualHost) hm := hmac.New(sha1.New, []byte(secretAccessKey)) hm.Write([]byte(stringToSign)) // Prepare auth header. authHeader := new(bytes.Buffer) fmt.Fprintf(authHeader, "%s %s:", signV2Algorithm, accessKeyID) encoder := base64.NewEncoder(base64.StdEncoding, authHeader) encoder.Write(hm.Sum(nil)) encoder.Close() // Set Authorization header. req.Header.Set("Authorization", authHeader.String()) return &req } // From the Amazon docs: // // StringToSign = HTTP-Verb + "\n" + // // Content-Md5 + "\n" + // Content-Type + "\n" + // Expires + "\n" + // CanonicalizedProtocolHeaders + // CanonicalizedResource; func preStringToSignV2(req http.Request, virtualHost bool) string { buf := new(bytes.Buffer) // Write standard headers. writePreSignV2Headers(buf, req) // Write canonicalized protocol headers if any. writeCanonicalizedHeaders(buf, req) // Write canonicalized Query resources if any. writeCanonicalizedResource(buf, req, virtualHost) return buf.String() } // writePreSignV2Headers - write preSign v2 required headers. func writePreSignV2Headers(buf *bytes.Buffer, req http.Request) { buf.WriteString(req.Method + "\n") buf.WriteString(req.Header.Get("Content-Md5") + "\n") buf.WriteString(req.Header.Get("Content-Type") + "\n") buf.WriteString(req.Header.Get("Expires") + "\n") } // From the Amazon docs: // // StringToSign = HTTP-Verb + "\n" + // // Content-Md5 + "\n" + // Content-Type + "\n" + // Date + "\n" + // CanonicalizedProtocolHeaders + // CanonicalizedResource; func stringToSignV2(req http.Request, virtualHost bool) string { buf := new(bytes.Buffer) // Write standard headers. writeSignV2Headers(buf, req) // Write canonicalized protocol headers if any. writeCanonicalizedHeaders(buf, req) // Write canonicalized Query resources if any. writeCanonicalizedResource(buf, req, virtualHost) return buf.String() } // writeSignV2Headers - write signV2 required headers. func writeSignV2Headers(buf *bytes.Buffer, req http.Request) { buf.WriteString(req.Method + "\n") buf.WriteString(req.Header.Get("Content-Md5") + "\n") buf.WriteString(req.Header.Get("Content-Type") + "\n") buf.WriteString(req.Header.Get("Date") + "\n") } // writeCanonicalizedHeaders - write canonicalized headers. func writeCanonicalizedHeaders(buf *bytes.Buffer, req http.Request) { var protoHeaders []string vals := make(map[string][]string) for k, vv := range req.Header { // All the AMZ headers should be lowercase lk := strings.ToLower(k) if strings.HasPrefix(lk, "x-amz") { protoHeaders = append(protoHeaders, lk) vals[lk] = vv } } sort.Strings(protoHeaders) for _, k := range protoHeaders { buf.WriteString(k) buf.WriteByte(':') for idx, v := range vals[k] { if idx > 0 { buf.WriteByte(',') } buf.WriteString(v) } buf.WriteByte('\n') } } // AWS S3 Signature V2 calculation rule is give here: // http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationStringToSign // Whitelist resource list that will be used in query string for signature-V2 calculation. // // This list should be kept alphabetically sorted, do not hastily edit. var resourceList = []string{ "acl", "cors", "delete", "encryption", "legal-hold", "lifecycle", "location", "logging", "notification", "partNumber", "policy", "replication", "requestPayment", "response-cache-control", "response-content-disposition", "response-content-encoding", "response-content-language", "response-content-type", "response-expires", "retention", "select", "select-type", "tagging", "torrent", "uploadId", "uploads", "versionId", "versioning", "versions", "website", } // From the Amazon docs: // // CanonicalizedResource = [ "/" + Bucket ] + // // + // [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"]; func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request, virtualHost bool) { // Save request URL. requestURL := req.URL // Get encoded URL path. buf.WriteString(encodeURL2Path(&req, virtualHost)) if requestURL.RawQuery != "" { var n int vals, _ := url.ParseQuery(requestURL.RawQuery) // Verify if any sub resource queries are present, if yes // canonicallize them. for _, resource := range resourceList { if vv, ok := vals[resource]; ok && len(vv) > 0 { n++ // First element switch n { case 1: buf.WriteByte('?') // The rest default: buf.WriteByte('&') } buf.WriteString(resource) // Request parameters if len(vv[0]) > 0 { buf.WriteByte('=') buf.WriteString(vv[0]) } } } } } minio-go-7.0.97/pkg/signer/request-signature-v2_test.go000066400000000000000000000022151510244170000230000ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package signer import ( "sort" "testing" ) // Tests for 'func TestResourceListSorting(t *testing.T)'. func TestResourceListSorting(t *testing.T) { sortedResourceList := make([]string, len(resourceList)) copy(sortedResourceList, resourceList) sort.Strings(sortedResourceList) for i := 0; i < len(resourceList); i++ { if resourceList[i] != sortedResourceList[i] { t.Errorf("Expected resourceList[%d] = \"%s\", resourceList is not correctly sorted.", i, sortedResourceList[i]) break } } } minio-go-7.0.97/pkg/signer/request-signature-v4.go000066400000000000000000000307511510244170000217510ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package signer import ( "bytes" "encoding/hex" "net/http" "sort" "strconv" "strings" "time" "github.com/minio/minio-go/v7/pkg/s3utils" ) // Signature and API related constants. const ( signV4Algorithm = "AWS4-HMAC-SHA256" iso8601DateFormat = "20060102T150405Z" yyyymmdd = "20060102" ) // Different service types const ( ServiceTypeS3 = "s3" ServiceTypeSTS = "sts" ServiceTypeS3Express = "s3express" ) // Excerpts from @lsegal - // https:/github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258. // // * User-Agent // This is ignored from signing because signing this causes problems with generating pre-signed // URLs (that are executed by other agents) or when customers pass requests through proxies, which // may modify the user-agent. // // * Authorization // Is skipped for obvious reasons. // // * Accept-Encoding // Some S3 servers like Hitachi Content Platform do not honor this header for signature // calculation. var v4IgnoredHeaders = map[string]bool{ "Accept-Encoding": true, "Authorization": true, "User-Agent": true, } // getSigningKey hmac seed to calculate final signature. func getSigningKey(secret, loc string, t time.Time, serviceType string) []byte { date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd))) location := sumHMAC(date, []byte(loc)) service := sumHMAC(location, []byte(serviceType)) signingKey := sumHMAC(service, []byte("aws4_request")) return signingKey } // getSignature final signature in hexadecimal form. func getSignature(signingKey []byte, stringToSign string) string { return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) } // getScope generate a string of a specific date, an AWS region, and a // service. func getScope(location string, t time.Time, serviceType string) string { scope := strings.Join([]string{ t.Format(yyyymmdd), location, serviceType, "aws4_request", }, "/") return scope } // GetCredential generate a credential string. func GetCredential(accessKeyID, location string, t time.Time, serviceType string) string { scope := getScope(location, t, serviceType) return accessKeyID + "/" + scope } // getHashedPayload get the hexadecimal value of the SHA256 hash of // the request payload. func getHashedPayload(req http.Request) string { hashedPayload := req.Header.Get("X-Amz-Content-Sha256") if hashedPayload == "" { // Presign does not have a payload, use S3 recommended value. hashedPayload = unsignedPayload } return hashedPayload } // getCanonicalHeaders generate a list of request headers for // signature. func getCanonicalHeaders(req http.Request, ignoredHeaders map[string]bool) string { var headers []string vals := make(map[string][]string) for k, vv := range req.Header { if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok { continue // ignored header } headers = append(headers, strings.ToLower(k)) vals[strings.ToLower(k)] = vv } if !headerExists("host", headers) { headers = append(headers, "host") } sort.Strings(headers) var buf bytes.Buffer // Save all the headers in canonical form
: newline // separated for each header. for _, k := range headers { buf.WriteString(k) buf.WriteByte(':') switch k { case "host": buf.WriteString(getHostAddr(&req)) buf.WriteByte('\n') default: for idx, v := range vals[k] { if idx > 0 { buf.WriteByte(',') } buf.WriteString(signV4TrimAll(v)) } buf.WriteByte('\n') } } return buf.String() } func headerExists(key string, headers []string) bool { for _, k := range headers { if k == key { return true } } return false } // getSignedHeaders generate all signed request headers. // i.e lexically sorted, semicolon-separated list of lowercase // request header names. func getSignedHeaders(req http.Request, ignoredHeaders map[string]bool) string { var headers []string for k := range req.Header { if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok { continue // Ignored header found continue. } headers = append(headers, strings.ToLower(k)) } if !headerExists("host", headers) { headers = append(headers, "host") } sort.Strings(headers) return strings.Join(headers, ";") } // getCanonicalRequest generate a canonical request of style. // // canonicalRequest = // // \n // \n // \n // \n // \n // func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool, hashedPayload string) string { req.URL.RawQuery = strings.ReplaceAll(req.URL.Query().Encode(), "+", "%20") canonicalRequest := strings.Join([]string{ req.Method, s3utils.EncodePath(req.URL.Path), req.URL.RawQuery, getCanonicalHeaders(req, ignoredHeaders), getSignedHeaders(req, ignoredHeaders), hashedPayload, }, "\n") return canonicalRequest } // getStringToSign a string based on selected query values. func getStringToSignV4(t time.Time, location, canonicalRequest, serviceType string) string { stringToSign := signV4Algorithm + "\n" + t.Format(iso8601DateFormat) + "\n" stringToSign = stringToSign + getScope(location, t, serviceType) + "\n" stringToSign += hex.EncodeToString(sum256([]byte(canonicalRequest))) return stringToSign } // PreSignV4 presign the request, in accordance with // http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html. func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, expires int64) *http.Request { // Presign is not needed for anonymous credentials. if accessKeyID == "" || secretAccessKey == "" { return &req } // Initial time. t := time.Now().UTC() // Get credential string. credential := GetCredential(accessKeyID, location, t, ServiceTypeS3) // Get all signed headers. signedHeaders := getSignedHeaders(req, v4IgnoredHeaders) // Set URL query. query := req.URL.Query() query.Set("X-Amz-Algorithm", signV4Algorithm) query.Set("X-Amz-Date", t.Format(iso8601DateFormat)) query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10)) query.Set("X-Amz-SignedHeaders", signedHeaders) query.Set("X-Amz-Credential", credential) // Set session token if available. if sessionToken != "" { if v := req.Header.Get("x-amz-s3session-token"); v != "" { query.Set("X-Amz-S3session-Token", sessionToken) } else { query.Set("X-Amz-Security-Token", sessionToken) } } req.URL.RawQuery = query.Encode() // Get canonical request. canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, getHashedPayload(req)) // Get string to sign from canonical request. stringToSign := getStringToSignV4(t, location, canonicalRequest, ServiceTypeS3) // Gext hmac signing key. signingKey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3) // Calculate signature. signature := getSignature(signingKey, stringToSign) // Add signature header to RawQuery. req.URL.RawQuery += "&X-Amz-Signature=" + signature return &req } // PostPresignSignatureV4 - presigned signature for PostPolicy // requests. func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string { // Get signining key. signingkey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3) // Calculate signature. signature := getSignature(signingkey, policyBase64) return signature } // SignV4STS - signature v4 for STS request. func SignV4STS(req http.Request, accessKeyID, secretAccessKey, location string) *http.Request { return signV4(req, accessKeyID, secretAccessKey, "", location, ServiceTypeSTS, nil) } // Internal function called for different service types. func signV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location, serviceType string, trailer http.Header) *http.Request { // Signature calculation is not needed for anonymous credentials. if accessKeyID == "" || secretAccessKey == "" { return &req } // Initial time. t := time.Now().UTC() // Set x-amz-date. req.Header.Set("X-Amz-Date", t.Format(iso8601DateFormat)) // Set session token if available. if sessionToken != "" { // S3 Express token if not set then set sessionToken // with older x-amz-security-token header. if v := req.Header.Get("x-amz-s3session-token"); v == "" { req.Header.Set("X-Amz-Security-Token", sessionToken) } } if len(trailer) > 0 { for k := range trailer { req.Header.Add("X-Amz-Trailer", strings.ToLower(k)) } req.Header.Set("Content-Encoding", "aws-chunked") req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(req.ContentLength, 10)) } hashedPayload := getHashedPayload(req) if serviceType == ServiceTypeSTS { // Content sha256 header is not sent with the request // but it is expected to have sha256 of payload for signature // in STS service type request. req.Header.Del("X-Amz-Content-Sha256") } // Get canonical request. canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, hashedPayload) // Get string to sign from canonical request. stringToSign := getStringToSignV4(t, location, canonicalRequest, serviceType) // Get hmac signing key. signingKey := getSigningKey(secretAccessKey, location, t, serviceType) // Get credential string. credential := GetCredential(accessKeyID, location, t, serviceType) // Get all signed headers. signedHeaders := getSignedHeaders(req, v4IgnoredHeaders) // Calculate signature. signature := getSignature(signingKey, stringToSign) // If regular request, construct the final authorization header. parts := []string{ signV4Algorithm + " Credential=" + credential, "SignedHeaders=" + signedHeaders, "Signature=" + signature, } // Set authorization header. auth := strings.Join(parts, ", ") req.Header.Set("Authorization", auth) if len(trailer) > 0 { // Use custom chunked encoding. req.Trailer = trailer return StreamingUnsignedV4(&req, sessionToken, req.ContentLength, t) } return &req } // UnsignedTrailer will do chunked encoding with a custom trailer. func UnsignedTrailer(req http.Request, trailer http.Header) *http.Request { if len(trailer) == 0 { return &req } // Initial time. t := time.Now().UTC() // Set x-amz-date. req.Header.Set("X-Amz-Date", t.Format(iso8601DateFormat)) for k := range trailer { req.Header.Add("X-Amz-Trailer", strings.ToLower(k)) } req.Header.Set("Content-Encoding", "aws-chunked") req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(req.ContentLength, 10)) // Use custom chunked encoding. req.Trailer = trailer return StreamingUnsignedV4(&req, "", req.ContentLength, t) } // SignV4 sign the request before Do(), in accordance with // http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html. func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request { return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3, nil) } // SignV4Express sign the request before Do(), in accordance with // http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html. func SignV4Express(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request { return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3Express, nil) } // SignV4TrailerExpress sign the request before Do(), in accordance with // http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html func SignV4TrailerExpress(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, trailer http.Header) *http.Request { return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3Express, trailer) } // SignV4Trailer sign the request before Do(), in accordance with // http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html func SignV4Trailer(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, trailer http.Header) *http.Request { return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3, trailer) } minio-go-7.0.97/pkg/signer/request-signature-v4_test.go000066400000000000000000000035121510244170000230030ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package signer import ( "fmt" "io" "net/http" "strings" "testing" ) func TestRequestHost(t *testing.T) { req, _ := buildRequest("dynamodb", "us-east-1", "{}") req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a" req.Host = "myhost" canonicalHeaders := getCanonicalHeaders(*req, v4IgnoredHeaders) if !strings.Contains(canonicalHeaders, "host:"+req.Host) { t.Errorf("canonical host header invalid") } } func buildRequest(serviceName, region, body string) (*http.Request, io.ReadSeeker) { endpoint := "https://" + serviceName + "." + region + ".amazonaws.com" reader := strings.NewReader(body) req, _ := http.NewRequest(http.MethodPost, endpoint, reader) req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()" req.Header.Add("Accept-Encoding", "identity") req.Header.Add("Content-Type", "application/x-amz-json-1.0") req.Header.Add("Content-Length", fmt.Sprint(len(body))) req.Header.Add("X-Amz-Meta-Other-Header", "some-value=!@#$%^&* (+)") req.Header.Add("X-Amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)") req.Header.Add("X-amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)") req.Header.Add("X-Amz-Target", "prefix.Operation") return req, reader } minio-go-7.0.97/pkg/signer/request-signature_test.go000066400000000000000000000056721510244170000224650ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package signer import ( "net/http" "strings" "testing" ) // Tests signature calculation. func TestSignatureCalculationV4(t *testing.T) { req, err := http.NewRequest(http.MethodGet, "https://s3.amazonaws.com", nil) if err != nil { t.Fatal("Error:", err) } req = SignV4(*req, "", "", "", "us-east-1") if req.Header.Get("Authorization") != "" { t.Fatal("Error: anonymous credentials should not have Authorization header.") } req = PreSignV4(*req, "", "", "", "us-east-1", 0) if strings.Contains(req.URL.RawQuery, "X-Amz-Signature") { t.Fatal("Error: anonymous credentials should not have Signature query resource.") } req = SignV4(*req, "ACCESS-KEY", "SECRET-KEY", "", "us-east-1") if req.Header.Get("Authorization") == "" { t.Fatal("Error: normal credentials should have Authorization header.") } req = PreSignV4(*req, "ACCESS-KEY", "SECRET-KEY", "", "us-east-1", 0) if !strings.Contains(req.URL.RawQuery, "X-Amz-Signature") { t.Fatal("Error: normal credentials should have Signature query resource.") } } func TestSignatureCalculationV2(t *testing.T) { testCases := []struct { endpointURL string virtualHost bool }{ {endpointURL: "https://s3.amazonaws.com/", virtualHost: false}, {endpointURL: "https://testbucket.s3.amazonaws.com/", virtualHost: true}, } for i, testCase := range testCases { req, err := http.NewRequest(http.MethodGet, testCase.endpointURL, nil) if err != nil { t.Fatalf("Test %d, Error: %v", i+1, err) } req = SignV2(*req, "", "", testCase.virtualHost) if req.Header.Get("Authorization") != "" { t.Fatalf("Test %d, Error: anonymous credentials should not have Authorization header.", i+1) } req = PreSignV2(*req, "", "", 0, testCase.virtualHost) if strings.Contains(req.URL.RawQuery, "Signature") { t.Fatalf("Test %d, Error: anonymous credentials should not have Signature query resource.", i+1) } req = SignV2(*req, "ACCESS-KEY", "SECRET-KEY", testCase.virtualHost) if req.Header.Get("Authorization") == "" { t.Fatalf("Test %d, Error: normal credentials should have Authorization header.", i+1) } req = PreSignV2(*req, "ACCESS-KEY", "SECRET-KEY", 0, testCase.virtualHost) if !strings.Contains(req.URL.RawQuery, "Signature") { t.Fatalf("Test %d, Error: normal credentials should not have Signature query resource.", i+1) } } } minio-go-7.0.97/pkg/signer/test-utils_test.go000066400000000000000000000056041510244170000211060ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package signer import ( "bufio" "bytes" "crypto/tls" "io" "net/http" "strings" ) // N B minio-go should compile on go1.5.3 onwards and httptest package is // available only from go.1.7.x. The following function is taken from // Go httptest package to be able to build on older versions of Go. // NewRequest returns a new incoming server Request, suitable // for passing to an http.Handler for testing. // // The target is the RFC 7230 "request-target": it may be either a // path or an absolute URL. If target is an absolute URL, the host name // from the URL is used. Otherwise, "example.com" is used. // // The TLS field is set to a non-nil dummy value if target has scheme // "https". // // The Request.Proto is always HTTP/1.1. // // An empty method means http.MethodGet. // // The provided body may be nil. If the body is of type *bytes.Reader, // *strings.Reader, or *bytes.Buffer, the Request.ContentLength is // set. // // NewRequest panics on error for ease of use in testing, where a // panic is acceptable. func NewRequest(method, target string, body io.Reader) *http.Request { if method == "" { method = http.MethodGet } req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(method + " " + target + " HTTP/1.0\r\n\r\n"))) if err != nil { panic("invalid NewRequest arguments; " + err.Error()) } // HTTP/1.0 was used above to avoid needing a Host field. Change it to 1.1 here. req.Proto = "HTTP/1.1" req.ProtoMinor = 1 req.Close = false if body != nil { switch v := body.(type) { case *bytes.Buffer: req.ContentLength = int64(v.Len()) case *bytes.Reader: req.ContentLength = int64(v.Len()) case *strings.Reader: req.ContentLength = int64(v.Len()) default: req.ContentLength = -1 } if rc, ok := body.(io.ReadCloser); ok { req.Body = rc } else { req.Body = io.NopCloser(body) } } // 192.0.2.0/24 is "TEST-NET" in RFC 5737 for use solely in // documentation and example source code and should not be // used publicly. req.RemoteAddr = "192.0.2.1:1234" if req.Host == "" { req.Host = "example.com" } if strings.HasPrefix(target, "https://") { req.TLS = &tls.ConnectionState{ Version: tls.VersionTLS12, HandshakeComplete: true, ServerName: req.Host, } } return req } minio-go-7.0.97/pkg/signer/utils.go000066400000000000000000000035131510244170000170670ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package signer import ( "crypto/hmac" "crypto/sha256" "net/http" "strings" ) // unsignedPayload - value to be set to X-Amz-Content-Sha256 header when const unsignedPayload = "UNSIGNED-PAYLOAD" // sum256 calculate sha256 sum for an input byte array. func sum256(data []byte) []byte { hash := sha256.New() hash.Write(data) return hash.Sum(nil) } // sumHMAC calculate hmac between two input byte array. func sumHMAC(key, data []byte) []byte { hash := hmac.New(sha256.New, key) hash.Write(data) return hash.Sum(nil) } // getHostAddr returns host header if available, otherwise returns host from URL func getHostAddr(req *http.Request) string { host := req.Header.Get("host") if host != "" && req.Host != host { return host } if req.Host != "" { return req.Host } return req.URL.Host } // Trim leading and trailing spaces and replace sequential spaces with one space, following Trimall() // in http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html func signV4TrimAll(input string) string { // Compress adjacent spaces (a space is determined by // unicode.IsSpace() internally here) to one space and return return strings.Join(strings.Fields(input), " ") } minio-go-7.0.97/pkg/signer/utils_test.go000066400000000000000000000057741510244170000201410ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package signer import ( "fmt" "net/http" "net/url" "testing" ) // Tests url encoding. func TestEncodeURL2Path(t *testing.T) { type urlStrings struct { virtualHost bool bucketName string objName string encodedObjName string } want := []urlStrings{ { virtualHost: true, bucketName: "bucketName", objName: "本語", encodedObjName: "%E6%9C%AC%E8%AA%9E", }, { virtualHost: true, bucketName: "bucketName", objName: "本語.1", encodedObjName: "%E6%9C%AC%E8%AA%9E.1", }, { virtualHost: true, objName: ">123>3123123", bucketName: "bucketName", encodedObjName: "%3E123%3E3123123", }, { virtualHost: true, bucketName: "bucketName", objName: "test 1 2.txt", encodedObjName: "test%201%202.txt", }, { virtualHost: false, bucketName: "test.bucketName", objName: "test++ 1.txt", encodedObjName: "test%2B%2B%201.txt", }, } for i, o := range want { var hostURL string if o.virtualHost { hostURL = fmt.Sprintf("https://%s.s3.amazonaws.com/%s", o.bucketName, o.objName) } else { hostURL = fmt.Sprintf("https://s3.amazonaws.com/%s/%s", o.bucketName, o.objName) } u, err := url.Parse(hostURL) if err != nil { t.Fatalf("Test %d, Error: %v", i+1, err) } expectedPath := "/" + o.bucketName + "/" + o.encodedObjName if foundPath := encodeURL2Path(&http.Request{URL: u}, o.virtualHost); foundPath != expectedPath { t.Fatalf("Test %d, Error: expected = `%v`, found = `%v`", i+1, expectedPath, foundPath) } } } // TestSignV4TrimAll - tests the logic of TrimAll() function func TestSignV4TrimAll(t *testing.T) { testCases := []struct { // Input. inputStr string // Expected result. result string }{ {"本語", "本語"}, {" abc ", "abc"}, {" a b ", "a b"}, {"a b ", "a b"}, {"a b", "a b"}, {"a b", "a b"}, {" a b c ", "a b c"}, {"a \t b c ", "a b c"}, {"\"a \t b c ", "\"a b c"}, {" \t\n\u000b\r\fa \t\n\u000b\r\f b \t\n\u000b\r\f c \t\n\u000b\r\f", "a b c"}, } // Tests generated values from url encoded name. for i, testCase := range testCases { result := signV4TrimAll(testCase.inputStr) if testCase.result != result { t.Errorf("Test %d: Expected signV4TrimAll result to be \"%s\", but found it to be \"%s\" instead", i+1, testCase.result, result) } } } minio-go-7.0.97/pkg/singleflight/000077500000000000000000000000001510244170000165665ustar00rootroot00000000000000minio-go-7.0.97/pkg/singleflight/singleflight.go000066400000000000000000000126101510244170000215740ustar00rootroot00000000000000// Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package singleflight provides a duplicate function call suppression // mechanism. // This is forked to provide type safety and have non-string keys. package singleflight import ( "bytes" "errors" "fmt" "runtime" "runtime/debug" "sync" ) // errGoexit indicates the runtime.Goexit was called in // the user given function. var errGoexit = errors.New("runtime.Goexit was called") // A panicError is an arbitrary value recovered from a panic // with the stack trace during the execution of given function. type panicError struct { value interface{} stack []byte } // Error implements error interface. func (p *panicError) Error() string { return fmt.Sprintf("%v\n\n%s", p.value, p.stack) } func (p *panicError) Unwrap() error { err, ok := p.value.(error) if !ok { return nil } return err } func newPanicError(v interface{}) error { stack := debug.Stack() // The first line of the stack trace is of the form "goroutine N [status]:" // but by the time the panic reaches Do the goroutine may no longer exist // and its status will have changed. Trim out the misleading line. if line := bytes.IndexByte(stack, '\n'); line >= 0 { stack = stack[line+1:] } return &panicError{value: v, stack: stack} } // call is an in-flight or completed singleflight.Do call type call[V any] struct { wg sync.WaitGroup // These fields are written once before the WaitGroup is done // and are only read after the WaitGroup is done. val V err error // These fields are read and written with the singleflight // mutex held before the WaitGroup is done, and are read but // not written after the WaitGroup is done. dups int chans []chan<- Result[V] } // Group represents a class of work and forms a namespace in // which units of work can be executed with duplicate suppression. type Group[K comparable, V any] struct { mu sync.Mutex // protects m m map[K]*call[V] // lazily initialized } // Result holds the results of Do, so they can be passed // on a channel. type Result[V any] struct { Val V Err error Shared bool } // Do executes and returns the results of the given function, making // sure that only one execution is in-flight for a given key at a // time. If a duplicate comes in, the duplicate caller waits for the // original to complete and receives the same results. // The return value shared indicates whether v was given to multiple callers. // //nolint:revive func (g *Group[K, V]) Do(key K, fn func() (V, error)) (v V, err error, shared bool) { g.mu.Lock() if g.m == nil { g.m = make(map[K]*call[V]) } if c, ok := g.m[key]; ok { c.dups++ g.mu.Unlock() c.wg.Wait() if e, ok := c.err.(*panicError); ok { panic(e) } else if c.err == errGoexit { runtime.Goexit() } return c.val, c.err, true } c := new(call[V]) c.wg.Add(1) g.m[key] = c g.mu.Unlock() g.doCall(c, key, fn) return c.val, c.err, c.dups > 0 } // DoChan is like Do but returns a channel that will receive the // results when they are ready. // // The returned channel will not be closed. func (g *Group[K, V]) DoChan(key K, fn func() (V, error)) <-chan Result[V] { ch := make(chan Result[V], 1) g.mu.Lock() if g.m == nil { g.m = make(map[K]*call[V]) } if c, ok := g.m[key]; ok { c.dups++ c.chans = append(c.chans, ch) g.mu.Unlock() return ch } c := &call[V]{chans: []chan<- Result[V]{ch}} c.wg.Add(1) g.m[key] = c g.mu.Unlock() go g.doCall(c, key, fn) return ch } // doCall handles the single call for a key. func (g *Group[K, V]) doCall(c *call[V], key K, fn func() (V, error)) { normalReturn := false recovered := false // use double-defer to distinguish panic from runtime.Goexit, // more details see https://golang.org/cl/134395 defer func() { // the given function invoked runtime.Goexit if !normalReturn && !recovered { c.err = errGoexit } g.mu.Lock() defer g.mu.Unlock() c.wg.Done() if g.m[key] == c { delete(g.m, key) } if e, ok := c.err.(*panicError); ok { // In order to prevent the waiting channels from being blocked forever, // needs to ensure that this panic cannot be recovered. if len(c.chans) > 0 { go panic(e) select {} // Keep this goroutine around so that it will appear in the crash dump. } else { panic(e) } } else if c.err == errGoexit { // Already in the process of goexit, no need to call again } else { // Normal return for _, ch := range c.chans { ch <- Result[V]{c.val, c.err, c.dups > 0} } } }() func() { defer func() { if !normalReturn { // Ideally, we would wait to take a stack trace until we've determined // whether this is a panic or a runtime.Goexit. // // Unfortunately, the only way we can distinguish the two is to see // whether the recover stopped the goroutine from terminating, and by // the time we know that, the part of the stack trace relevant to the // panic has been discarded. if r := recover(); r != nil { c.err = newPanicError(r) } } }() c.val, c.err = fn() normalReturn = true }() if !normalReturn { recovered = true } } // Forget tells the singleflight to forget about a key. Future calls // to Do for this key will call the function rather than waiting for // an earlier call to complete. func (g *Group[K, V]) Forget(key K) { g.mu.Lock() delete(g.m, key) g.mu.Unlock() } minio-go-7.0.97/pkg/singleflight/singleflight_test.go000066400000000000000000000221541510244170000226370ustar00rootroot00000000000000// Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package singleflight import ( "bytes" "errors" "fmt" "os" "os/exec" "runtime" "runtime/debug" "strings" "sync" "sync/atomic" "testing" "time" ) type errValue struct{} func (err *errValue) Error() string { return "error value" } func TestPanicErrorUnwrap(t *testing.T) { t.Parallel() testCases := []struct { name string panicValue any wrappedErrorType bool }{ { name: "panicError wraps non-error type", panicValue: &panicError{value: "string value"}, wrappedErrorType: false, }, { name: "panicError wraps error type", panicValue: &panicError{value: new(errValue)}, wrappedErrorType: false, }, } for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() var recovered any group := &Group[string, any]{} func() { defer func() { recovered = recover() t.Logf("after panic(%#v) in group.Do, recovered %#v", tc.panicValue, recovered) }() _, _, _ = group.Do(tc.name, func() (any, error) { panic(tc.panicValue) }) }() if recovered == nil { t.Fatal("expected a non-nil panic value") } err, ok := recovered.(error) if !ok { t.Fatalf("recovered non-error type: %T", recovered) } if !errors.Is(err, new(errValue)) && tc.wrappedErrorType { t.Errorf("unexpected wrapped error type %T; want %T", err, new(errValue)) } }) } } func TestDo(t *testing.T) { var g Group[string, any] v, err, _ := g.Do("key", func() (any, error) { return "bar", nil }) if got, want := fmt.Sprintf("%v (%T)", v, v), "bar (string)"; got != want { t.Errorf("Do = %v; want %v", got, want) } if err != nil { t.Errorf("Do error = %v", err) } } func TestDoErr(t *testing.T) { var g Group[string, any] someErr := errors.New("Some error") v, err, _ := g.Do("key", func() (any, error) { return nil, someErr }) if err != someErr { t.Errorf("Do error = %v; want someErr %v", err, someErr) } if v != nil { t.Errorf("unexpected non-nil value %#v", v) } } func TestDoDupSuppress(t *testing.T) { var g Group[string, any] var wg1, wg2 sync.WaitGroup c := make(chan string, 1) var calls int32 fn := func() (any, error) { if atomic.AddInt32(&calls, 1) == 1 { // First invocation. wg1.Done() } v := <-c c <- v // pump; make available for any future calls time.Sleep(10 * time.Millisecond) // let more goroutines enter Do return v, nil } const n = 10 wg1.Add(1) for i := 0; i < n; i++ { wg1.Add(1) wg2.Add(1) go func() { defer wg2.Done() wg1.Done() v, err, _ := g.Do("key", fn) if err != nil { t.Errorf("Do error: %v", err) return } if s, _ := v.(string); s != "bar" { t.Errorf("Do = %T %v; want %q", v, v, "bar") } }() } wg1.Wait() // At least one goroutine is in fn now and all of them have at // least reached the line before the Do. c <- "bar" wg2.Wait() if got := atomic.LoadInt32(&calls); got <= 0 || got >= n { t.Errorf("number of calls = %d; want over 0 and less than %d", got, n) } } // Test that singleflight behaves correctly after Forget called. // See https://github.com/golang/go/issues/31420 func TestForget(t *testing.T) { var g Group[string, any] var ( firstStarted = make(chan struct{}) unblockFirst = make(chan struct{}) firstFinished = make(chan struct{}) ) go func() { g.Do("key", func() (i any, e error) { close(firstStarted) <-unblockFirst close(firstFinished) return i, e }) }() <-firstStarted g.Forget("key") unblockSecond := make(chan struct{}) secondResult := g.DoChan("key", func() (i any, e error) { <-unblockSecond return 2, nil }) close(unblockFirst) <-firstFinished thirdResult := g.DoChan("key", func() (i any, e error) { return 3, nil }) close(unblockSecond) <-secondResult r := <-thirdResult if r.Val != 2 { t.Errorf("We should receive result produced by second call, expected: 2, got %d", r.Val) } } func TestDoChan(t *testing.T) { var g Group[string, any] ch := g.DoChan("key", func() (any, error) { return "bar", nil }) res := <-ch v := res.Val err := res.Err if got, want := fmt.Sprintf("%v (%T)", v, v), "bar (string)"; got != want { t.Errorf("Do = %v; want %v", got, want) } if err != nil { t.Errorf("Do error = %v", err) } } // Test singleflight behaves correctly after Do panic. // See https://github.com/golang/go/issues/41133 func TestPanicDo(t *testing.T) { var g Group[string, any] fn := func() (any, error) { panic("invalid memory address or nil pointer dereference") } const n = 5 waited := int32(n) panicCount := int32(0) done := make(chan struct{}) for i := 0; i < n; i++ { go func() { defer func() { if err := recover(); err != nil { t.Logf("Got panic: %v\n%s", err, debug.Stack()) atomic.AddInt32(&panicCount, 1) } if atomic.AddInt32(&waited, -1) == 0 { close(done) } }() g.Do("key", fn) }() } select { case <-done: if panicCount != n { t.Errorf("Expect %d panic, but got %d", n, panicCount) } case <-time.After(time.Second): t.Fatalf("Do hangs") } } func TestGoexitDo(t *testing.T) { var g Group[string, any] fn := func() (any, error) { runtime.Goexit() return nil, nil } const n = 5 waited := int32(n) done := make(chan struct{}) for i := 0; i < n; i++ { go func() { var err error defer func() { if err != nil { t.Errorf("Error should be nil, but got: %v", err) } if atomic.AddInt32(&waited, -1) == 0 { close(done) } }() _, err, _ = g.Do("key", fn) }() } select { case <-done: case <-time.After(time.Second): t.Fatalf("Do hangs") } } func executable(t testing.TB) string { exe, err := os.Executable() if err != nil { t.Skipf("skipping: test executable not found") } // Control case: check whether exec.Command works at all. // (For example, it might fail with a permission error on iOS.) cmd := exec.Command(exe, "-test.list=^$") cmd.Env = []string{} if err := cmd.Run(); err != nil { t.Skipf("skipping: exec appears not to work on %s: %v", runtime.GOOS, err) } return exe } func TestPanicDoChan(t *testing.T) { if os.Getenv("TEST_PANIC_DOCHAN") != "" { defer func() { recover() }() g := new(Group[string, any]) ch := g.DoChan("", func() (any, error) { panic("Panicking in DoChan") }) <-ch t.Fatalf("DoChan unexpectedly returned") } t.Parallel() cmd := exec.Command(executable(t), "-test.run="+t.Name(), "-test.v") cmd.Env = append(os.Environ(), "TEST_PANIC_DOCHAN=1") out := new(bytes.Buffer) cmd.Stdout = out cmd.Stderr = out if err := cmd.Start(); err != nil { t.Fatal(err) } err := cmd.Wait() t.Logf("%s:\n%s", strings.Join(cmd.Args, " "), out) if err == nil { t.Errorf("Test subprocess passed; want a crash due to panic in DoChan") } if bytes.Contains(out.Bytes(), []byte("DoChan unexpectedly")) { t.Errorf("Test subprocess failed with an unexpected failure mode.") } if !bytes.Contains(out.Bytes(), []byte("Panicking in DoChan")) { t.Errorf("Test subprocess failed, but the crash isn't caused by panicking in DoChan") } } func TestPanicDoSharedByDoChan(t *testing.T) { if os.Getenv("TEST_PANIC_DOCHAN") != "" { blocked := make(chan struct{}) unblock := make(chan struct{}) g := new(Group[string, any]) go func() { defer func() { recover() }() g.Do("", func() (any, error) { close(blocked) <-unblock panic("Panicking in Do") }) }() <-blocked ch := g.DoChan("", func() (any, error) { panic("DoChan unexpectedly executed callback") }) close(unblock) <-ch t.Fatalf("DoChan unexpectedly returned") } t.Parallel() cmd := exec.Command(executable(t), "-test.run="+t.Name(), "-test.v") cmd.Env = append(os.Environ(), "TEST_PANIC_DOCHAN=1") out := new(bytes.Buffer) cmd.Stdout = out cmd.Stderr = out if err := cmd.Start(); err != nil { t.Fatal(err) } err := cmd.Wait() t.Logf("%s:\n%s", strings.Join(cmd.Args, " "), out) if err == nil { t.Errorf("Test subprocess passed; want a crash due to panic in Do shared by DoChan") } if bytes.Contains(out.Bytes(), []byte("DoChan unexpectedly")) { t.Errorf("Test subprocess failed with an unexpected failure mode.") } if !bytes.Contains(out.Bytes(), []byte("Panicking in Do")) { t.Errorf("Test subprocess failed, but the crash isn't caused by panicking in Do") } } func ExampleGroup() { g := new(Group[string, any]) block := make(chan struct{}) res1c := g.DoChan("key", func() (any, error) { <-block return "func 1", nil }) res2c := g.DoChan("key", func() (any, error) { <-block return "func 2", nil }) close(block) res1 := <-res1c res2 := <-res2c // Results are shared by functions executed with duplicate keys. fmt.Println("Shared:", res2.Shared) // Only the first function is executed: it is registered and started with "key", // and doesn't complete before the second function is registered with a duplicate key. fmt.Println("Equal results:", res1.Val.(string) == res2.Val.(string)) fmt.Println("Result:", res1.Val) // Output: // Shared: true // Equal results: true // Result: func 1 } minio-go-7.0.97/pkg/sse/000077500000000000000000000000001510244170000147015ustar00rootroot00000000000000minio-go-7.0.97/pkg/sse/sse.go000066400000000000000000000035271510244170000160310ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package sse import "encoding/xml" // ApplySSEByDefault defines default encryption configuration, KMS or SSE. To activate // KMS, SSEAlgoritm needs to be set to "aws:kms" // Minio currently does not support Kms. type ApplySSEByDefault struct { KmsMasterKeyID string `xml:"KMSMasterKeyID,omitempty"` SSEAlgorithm string `xml:"SSEAlgorithm"` } // Rule layer encapsulates default encryption configuration type Rule struct { Apply ApplySSEByDefault `xml:"ApplyServerSideEncryptionByDefault"` } // Configuration is the default encryption configuration structure type Configuration struct { XMLName xml.Name `xml:"ServerSideEncryptionConfiguration"` Rules []Rule `xml:"Rule"` } // NewConfigurationSSES3 initializes a new SSE-S3 configuration func NewConfigurationSSES3() *Configuration { return &Configuration{ Rules: []Rule{ { Apply: ApplySSEByDefault{ SSEAlgorithm: "AES256", }, }, }, } } // NewConfigurationSSEKMS initializes a new SSE-KMS configuration func NewConfigurationSSEKMS(kmsMasterKey string) *Configuration { return &Configuration{ Rules: []Rule{ { Apply: ApplySSEByDefault{ KmsMasterKeyID: kmsMasterKey, SSEAlgorithm: "aws:kms", }, }, }, } } minio-go-7.0.97/pkg/tags/000077500000000000000000000000001510244170000150455ustar00rootroot00000000000000minio-go-7.0.97/pkg/tags/tags.go000066400000000000000000000233421510244170000163360ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2020-2022 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package tags import ( "encoding/xml" "io" "net/url" "regexp" "sort" "strings" "unicode/utf8" ) // Error contains tag specific error. type Error interface { error Code() string } type errTag struct { code string message string } // Code contains error code. func (err errTag) Code() string { return err.code } // Error contains error message. func (err errTag) Error() string { return err.message } var ( errTooManyObjectTags = &errTag{"BadRequest", "Tags cannot be more than 10"} errTooManyTags = &errTag{"BadRequest", "Tags cannot be more than 50"} errInvalidTagKey = &errTag{"InvalidTag", "The TagKey you have provided is invalid"} errInvalidTagValue = &errTag{"InvalidTag", "The TagValue you have provided is invalid"} errDuplicateTagKey = &errTag{"InvalidTag", "Cannot provide multiple Tags with the same key"} ) // Tag comes with limitation as per // https://docs.aws.amazon.com/AmazonS3/latest/dev/object-tagging.html amd // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions const ( maxKeyLength = 128 maxValueLength = 256 maxObjectTagCount = 10 maxTagCount = 50 ) // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions // borrowed from this article and also testing various ASCII characters following regex // is supported by AWS S3 for both tags and values. var validTagKeyValue = regexp.MustCompile(`^[a-zA-Z0-9-+\-._:/@ =]+$`) func checkKey(key string) error { if len(key) == 0 { return errInvalidTagKey } if utf8.RuneCountInString(key) > maxKeyLength || !validTagKeyValue.MatchString(key) { return errInvalidTagKey } return nil } func checkValue(value string) error { if value != "" { if utf8.RuneCountInString(value) > maxValueLength || !validTagKeyValue.MatchString(value) { return errInvalidTagValue } } return nil } // Tag denotes key and value. type Tag struct { Key string `xml:"Key"` Value string `xml:"Value"` } func (tag Tag) String() string { return tag.Key + "=" + tag.Value } // IsEmpty returns whether this tag is empty or not. func (tag Tag) IsEmpty() bool { return tag.Key == "" } // Validate checks this tag. func (tag Tag) Validate() error { if err := checkKey(tag.Key); err != nil { return err } return checkValue(tag.Value) } // MarshalXML encodes to XML data. func (tag Tag) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if err := tag.Validate(); err != nil { return err } type subTag Tag // to avoid recursively calling MarshalXML() return e.EncodeElement(subTag(tag), start) } // UnmarshalXML decodes XML data to tag. func (tag *Tag) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { type subTag Tag // to avoid recursively calling UnmarshalXML() var st subTag if err := d.DecodeElement(&st, &start); err != nil { return err } if err := Tag(st).Validate(); err != nil { return err } *tag = Tag(st) return nil } // tagSet represents list of unique tags. type tagSet struct { tagMap map[string]string isObject bool } func (tags tagSet) String() string { if len(tags.tagMap) == 0 { return "" } var buf strings.Builder keys := make([]string, 0, len(tags.tagMap)) for k := range tags.tagMap { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { keyEscaped := url.QueryEscape(k) valueEscaped := url.QueryEscape(tags.tagMap[k]) if buf.Len() > 0 { buf.WriteByte('&') } buf.WriteString(keyEscaped) buf.WriteByte('=') buf.WriteString(valueEscaped) } return buf.String() } func (tags *tagSet) remove(key string) { delete(tags.tagMap, key) } func (tags *tagSet) set(key, value string, failOnExist bool) error { if failOnExist { if _, found := tags.tagMap[key]; found { return errDuplicateTagKey } } if err := checkKey(key); err != nil { return err } if err := checkValue(value); err != nil { return err } if tags.isObject { if len(tags.tagMap) == maxObjectTagCount { return errTooManyObjectTags } } else if len(tags.tagMap) == maxTagCount { return errTooManyTags } tags.tagMap[key] = value return nil } func (tags tagSet) count() int { return len(tags.tagMap) } func (tags tagSet) toMap() map[string]string { m := make(map[string]string, len(tags.tagMap)) for key, value := range tags.tagMap { m[key] = value } return m } // MarshalXML encodes to XML data. func (tags tagSet) MarshalXML(e *xml.Encoder, start xml.StartElement) error { tagList := struct { Tags []Tag `xml:"Tag"` }{} tagList.Tags = make([]Tag, 0, len(tags.tagMap)) for key, value := range tags.tagMap { tagList.Tags = append(tagList.Tags, Tag{key, value}) } return e.EncodeElement(tagList, start) } // UnmarshalXML decodes XML data to tag list. func (tags *tagSet) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { tagList := struct { Tags []Tag `xml:"Tag"` }{} if err := d.DecodeElement(&tagList, &start); err != nil { return err } if tags.isObject { if len(tagList.Tags) > maxObjectTagCount { return errTooManyObjectTags } } else if len(tagList.Tags) > maxTagCount { return errTooManyTags } m := make(map[string]string, len(tagList.Tags)) for _, tag := range tagList.Tags { if _, found := m[tag.Key]; found { return errDuplicateTagKey } m[tag.Key] = tag.Value } tags.tagMap = m return nil } type tagging struct { XMLName xml.Name `xml:"Tagging"` TagSet *tagSet `xml:"TagSet"` } // Tags is list of tags of XML request/response as per // https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketTagging.html#API_GetBucketTagging_RequestBody type Tags tagging func (tags Tags) String() string { return tags.TagSet.String() } // Remove removes a tag by its key. func (tags *Tags) Remove(key string) { tags.TagSet.remove(key) } // Set sets new tag. func (tags *Tags) Set(key, value string) error { return tags.TagSet.set(key, value, false) } // Count - return number of tags accounted for func (tags Tags) Count() int { return tags.TagSet.count() } // ToMap returns copy of tags. func (tags Tags) ToMap() map[string]string { return tags.TagSet.toMap() } // MapToObjectTags converts an input map of key and value into // *Tags data structure with validation. func MapToObjectTags(tagMap map[string]string) (*Tags, error) { return NewTags(tagMap, true) } // MapToBucketTags converts an input map of key and value into // *Tags data structure with validation. func MapToBucketTags(tagMap map[string]string) (*Tags, error) { return NewTags(tagMap, false) } // NewTags creates Tags from tagMap, If isObject is set, it validates for object tags. func NewTags(tagMap map[string]string, isObject bool) (*Tags, error) { tagging := &Tags{ TagSet: &tagSet{ tagMap: make(map[string]string), isObject: isObject, }, } for key, value := range tagMap { if err := tagging.TagSet.set(key, value, true); err != nil { return nil, err } } return tagging, nil } func unmarshalXML(reader io.Reader, isObject bool) (*Tags, error) { tagging := &Tags{ TagSet: &tagSet{ tagMap: make(map[string]string), isObject: isObject, }, } if err := xml.NewDecoder(reader).Decode(tagging); err != nil { return nil, err } return tagging, nil } // ParseBucketXML decodes XML data of tags in reader specified in // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketTagging.html#API_PutBucketTagging_RequestSyntax. func ParseBucketXML(reader io.Reader) (*Tags, error) { return unmarshalXML(reader, false) } // ParseObjectXML decodes XML data of tags in reader specified in // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectTagging.html#API_PutObjectTagging_RequestSyntax func ParseObjectXML(reader io.Reader) (*Tags, error) { return unmarshalXML(reader, true) } // stringsCut slices s around the first instance of sep, // returning the text before and after sep. // The found result reports whether sep appears in s. // If sep does not appear in s, cut returns s, "", false. func stringsCut(s, sep string) (before, after string, found bool) { if i := strings.Index(s, sep); i >= 0 { return s[:i], s[i+len(sep):], true } return s, "", false } func (tags *tagSet) parseTags(tgs string) (err error) { for tgs != "" { var key string key, tgs, _ = stringsCut(tgs, "&") if key == "" { continue } key, value, _ := stringsCut(key, "=") key, err1 := url.QueryUnescape(key) if err1 != nil { if err == nil { err = err1 } continue } value, err1 = url.QueryUnescape(value) if err1 != nil { if err == nil { err = err1 } continue } if err = tags.set(key, value, true); err != nil { return err } } return err } // Parse decodes HTTP query formatted string into tags which is limited by isObject. // A query formatted string is like "key1=value1&key2=value2". func Parse(s string, isObject bool) (*Tags, error) { tagging := &Tags{ TagSet: &tagSet{ tagMap: make(map[string]string), isObject: isObject, }, } if err := tagging.TagSet.parseTags(s); err != nil { return nil, err } return tagging, nil } // ParseObjectTags decodes HTTP query formatted string into tags. A query formatted string is like "key1=value1&key2=value2". func ParseObjectTags(s string) (*Tags, error) { return Parse(s, true) } minio-go-7.0.97/pkg/tags/tags_test.go000066400000000000000000000043011510244170000173670ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2022 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package tags import ( "fmt" "testing" ) func TestParseTags(t *testing.T) { testCases := []struct { tags string expectedErr bool count int }{ { "key1=value1&key2=value2", false, 2, }, { "store+forever=false&factory=true", false, 2, }, { " store forever =false&factory=true", false, 2, }, { "key=value=", false, 1, }, { fmt.Sprintf("%0128d=%0256d", 1, 1), false, 1, }, // Failure cases - https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions { "key1=value1&key1=value2", true, 0, }, { "key$=value1", true, 0, }, { "key1=value$", true, 0, }, { fmt.Sprintf("%0128d=%0257d", 1, 1), true, 0, }, { fmt.Sprintf("%0129d=%0256d", 1, 1), true, 0, }, { fmt.Sprintf("%0129d=%0257d", 1, 1), true, 0, }, } for _, testCase := range testCases { testCase := testCase t.Run(testCase.tags, func(t *testing.T) { tt, err := ParseObjectTags(testCase.tags) if !testCase.expectedErr && err != nil { t.Errorf("Expected success but failed with %v", err) } if testCase.expectedErr && err == nil { t.Error("Expected failure but found success") } if err == nil { if tt.Count() != testCase.count { t.Errorf("Expected count %d, got %d", testCase.count, tt.Count()) } else { t.Logf("%s", tt) } } }) } } func BenchmarkParseTags(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { ParseObjectTags("key1=value1&key2=value2") } } minio-go-7.0.97/post-policy.go000066400000000000000000000305321510244170000161420ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2023 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "encoding/base64" "errors" "fmt" "net/http" "strings" "time" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/tags" ) // expirationDateFormat date format for expiration key in json policy. const expirationDateFormat = "2006-01-02T15:04:05.000Z" // policyCondition explanation: // http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html // // Example: // // policyCondition { // matchType: "$eq", // key: "$Content-Type", // value: "image/png", // } type policyCondition struct { matchType string condition string value string } // PostPolicy - Provides strict static type conversion and validation // for Amazon S3's POST policy JSON string. type PostPolicy struct { // Expiration date and time of the POST policy. expiration time.Time // Collection of different policy conditions. conditions []policyCondition // ContentLengthRange minimum and maximum allowable size for the // uploaded content. contentLengthRange struct { min int64 max int64 } // Post form data. formData map[string]string } // NewPostPolicy - Instantiate new post policy. func NewPostPolicy() *PostPolicy { p := &PostPolicy{} p.conditions = make([]policyCondition, 0) p.formData = make(map[string]string) return p } // SetExpires - Sets expiration time for the new policy. func (p *PostPolicy) SetExpires(t time.Time) error { if t.IsZero() { return errInvalidArgument("No expiry time set.") } p.expiration = t return nil } // SetKey - Sets an object name for the policy based upload. func (p *PostPolicy) SetKey(key string) error { if strings.TrimSpace(key) == "" { return errInvalidArgument("Object name is empty.") } policyCond := policyCondition{ matchType: "eq", condition: "$key", value: key, } if err := p.addNewPolicy(policyCond); err != nil { return err } p.formData["key"] = key return nil } // SetKeyStartsWith - Sets an object name that an policy based upload // can start with. // Can use an empty value ("") to allow any key. func (p *PostPolicy) SetKeyStartsWith(keyStartsWith string) error { policyCond := policyCondition{ matchType: "starts-with", condition: "$key", value: keyStartsWith, } if err := p.addNewPolicy(policyCond); err != nil { return err } p.formData["key"] = keyStartsWith return nil } // SetBucket - Sets bucket at which objects will be uploaded to. func (p *PostPolicy) SetBucket(bucketName string) error { if strings.TrimSpace(bucketName) == "" { return errInvalidArgument("Bucket name is empty.") } policyCond := policyCondition{ matchType: "eq", condition: "$bucket", value: bucketName, } if err := p.addNewPolicy(policyCond); err != nil { return err } p.formData["bucket"] = bucketName return nil } // SetCondition - Sets condition for credentials, date and algorithm func (p *PostPolicy) SetCondition(matchType, condition, value string) error { if strings.TrimSpace(value) == "" { return errInvalidArgument("No value specified for condition") } policyCond := policyCondition{ matchType: matchType, condition: "$" + condition, value: value, } if condition == "X-Amz-Credential" || condition == "X-Amz-Date" || condition == "X-Amz-Algorithm" { if err := p.addNewPolicy(policyCond); err != nil { return err } p.formData[condition] = value return nil } return errInvalidArgument("Invalid condition in policy") } // SetTagging - Sets tagging for the object for this policy based upload. func (p *PostPolicy) SetTagging(tagging string) error { if strings.TrimSpace(tagging) == "" { return errInvalidArgument("No tagging specified.") } _, err := tags.ParseObjectXML(strings.NewReader(tagging)) if err != nil { return errors.New(s3ErrorResponseMap[MalformedXML]) //nolint } policyCond := policyCondition{ matchType: "eq", condition: "$tagging", value: tagging, } if err := p.addNewPolicy(policyCond); err != nil { return err } p.formData["tagging"] = tagging return nil } // SetContentType - Sets content-type of the object for this policy // based upload. func (p *PostPolicy) SetContentType(contentType string) error { if strings.TrimSpace(contentType) == "" { return errInvalidArgument("No content type specified.") } policyCond := policyCondition{ matchType: "eq", condition: "$Content-Type", value: contentType, } if err := p.addNewPolicy(policyCond); err != nil { return err } p.formData["Content-Type"] = contentType return nil } // SetContentTypeStartsWith - Sets what content-type of the object for this policy // based upload can start with. // Can use an empty value ("") to allow any content-type. func (p *PostPolicy) SetContentTypeStartsWith(contentTypeStartsWith string) error { policyCond := policyCondition{ matchType: "starts-with", condition: "$Content-Type", value: contentTypeStartsWith, } if err := p.addNewPolicy(policyCond); err != nil { return err } p.formData["Content-Type"] = contentTypeStartsWith return nil } // SetContentDisposition - Sets content-disposition of the object for this policy func (p *PostPolicy) SetContentDisposition(contentDisposition string) error { if strings.TrimSpace(contentDisposition) == "" { return errInvalidArgument("No content disposition specified.") } policyCond := policyCondition{ matchType: "eq", condition: "$Content-Disposition", value: contentDisposition, } if err := p.addNewPolicy(policyCond); err != nil { return err } p.formData["Content-Disposition"] = contentDisposition return nil } // SetContentEncoding - Sets content-encoding of the object for this policy func (p *PostPolicy) SetContentEncoding(contentEncoding string) error { if strings.TrimSpace(contentEncoding) == "" { return errInvalidArgument("No content encoding specified.") } policyCond := policyCondition{ matchType: "eq", condition: "$Content-Encoding", value: contentEncoding, } if err := p.addNewPolicy(policyCond); err != nil { return err } p.formData["Content-Encoding"] = contentEncoding return nil } // SetContentLengthRange - Set new min and max content length // condition for all incoming uploads. func (p *PostPolicy) SetContentLengthRange(minLen, maxLen int64) error { if minLen > maxLen { return errInvalidArgument("Minimum limit is larger than maximum limit.") } if minLen < 0 { return errInvalidArgument("Minimum limit cannot be negative.") } if maxLen <= 0 { return errInvalidArgument("Maximum limit cannot be non-positive.") } p.contentLengthRange.min = minLen p.contentLengthRange.max = maxLen return nil } // SetSuccessActionRedirect - Sets the redirect success url of the object for this policy // based upload. func (p *PostPolicy) SetSuccessActionRedirect(redirect string) error { if strings.TrimSpace(redirect) == "" { return errInvalidArgument("Redirect is empty") } policyCond := policyCondition{ matchType: "eq", condition: "$success_action_redirect", value: redirect, } if err := p.addNewPolicy(policyCond); err != nil { return err } p.formData["success_action_redirect"] = redirect return nil } // SetSuccessStatusAction - Sets the status success code of the object for this policy // based upload. func (p *PostPolicy) SetSuccessStatusAction(status string) error { if strings.TrimSpace(status) == "" { return errInvalidArgument("Status is empty") } policyCond := policyCondition{ matchType: "eq", condition: "$success_action_status", value: status, } if err := p.addNewPolicy(policyCond); err != nil { return err } p.formData["success_action_status"] = status return nil } // SetUserMetadata - Set user metadata as a key/value couple. // Can be retrieved through a HEAD request or an event. func (p *PostPolicy) SetUserMetadata(key, value string) error { if strings.TrimSpace(key) == "" { return errInvalidArgument("Key is empty") } if strings.TrimSpace(value) == "" { return errInvalidArgument("Value is empty") } headerName := fmt.Sprintf("x-amz-meta-%s", key) policyCond := policyCondition{ matchType: "eq", condition: fmt.Sprintf("$%s", headerName), value: value, } if err := p.addNewPolicy(policyCond); err != nil { return err } p.formData[headerName] = value return nil } // SetUserMetadataStartsWith - Set how an user metadata should starts with. // Can be retrieved through a HEAD request or an event. func (p *PostPolicy) SetUserMetadataStartsWith(key, value string) error { if strings.TrimSpace(key) == "" { return errInvalidArgument("Key is empty") } headerName := fmt.Sprintf("x-amz-meta-%s", key) policyCond := policyCondition{ matchType: "starts-with", condition: fmt.Sprintf("$%s", headerName), value: value, } if err := p.addNewPolicy(policyCond); err != nil { return err } p.formData[headerName] = value return nil } // SetChecksum sets the checksum of the request. func (p *PostPolicy) SetChecksum(c Checksum) error { if c.IsSet() { p.formData[amzChecksumAlgo] = c.Type.String() p.formData[c.Type.Key()] = c.Encoded() policyCond := policyCondition{ matchType: "eq", condition: fmt.Sprintf("$%s", amzChecksumAlgo), value: c.Type.String(), } if err := p.addNewPolicy(policyCond); err != nil { return err } policyCond = policyCondition{ matchType: "eq", condition: fmt.Sprintf("$%s", c.Type.Key()), value: c.Encoded(), } if err := p.addNewPolicy(policyCond); err != nil { return err } } return nil } // SetEncryption - sets encryption headers for POST API func (p *PostPolicy) SetEncryption(sse encrypt.ServerSide) { if sse == nil { return } h := http.Header{} sse.Marshal(h) for k, v := range h { p.formData[k] = v[0] } } // SetUserData - Set user data as a key/value couple. // Can be retrieved through a HEAD request or an event. func (p *PostPolicy) SetUserData(key, value string) error { if key == "" { return errInvalidArgument("Key is empty") } if value == "" { return errInvalidArgument("Value is empty") } headerName := fmt.Sprintf("x-amz-%s", key) policyCond := policyCondition{ matchType: "eq", condition: fmt.Sprintf("$%s", headerName), value: value, } if err := p.addNewPolicy(policyCond); err != nil { return err } p.formData[headerName] = value return nil } // addNewPolicy - internal helper to validate adding new policies. // Can use starts-with with an empty value ("") to allow any content within a form field. func (p *PostPolicy) addNewPolicy(policyCond policyCondition) error { if policyCond.matchType == "" || policyCond.condition == "" { return errInvalidArgument("Policy fields are empty.") } if policyCond.matchType != "starts-with" && policyCond.value == "" { return errInvalidArgument("Policy value is empty.") } p.conditions = append(p.conditions, policyCond) return nil } // String function for printing policy in json formatted string. func (p PostPolicy) String() string { return string(p.marshalJSON()) } // marshalJSON - Provides Marshaled JSON in bytes. func (p PostPolicy) marshalJSON() []byte { expirationStr := `"expiration":"` + p.expiration.UTC().Format(expirationDateFormat) + `"` var conditionsStr string conditions := []string{} for _, po := range p.conditions { conditions = append(conditions, fmt.Sprintf("[\"%s\",\"%s\",\"%s\"]", po.matchType, po.condition, po.value)) } if p.contentLengthRange.min != 0 || p.contentLengthRange.max != 0 { conditions = append(conditions, fmt.Sprintf("[\"content-length-range\", %d, %d]", p.contentLengthRange.min, p.contentLengthRange.max)) } if len(conditions) > 0 { conditionsStr = `"conditions":[` + strings.Join(conditions, ",") + "]" } retStr := "{" retStr = retStr + expirationStr + "," retStr += conditionsStr retStr += "}" return []byte(retStr) } // base64 - Produces base64 of PostPolicy's Marshaled json. func (p PostPolicy) base64() string { return base64.StdEncoding.EncodeToString(p.marshalJSON()) } minio-go-7.0.97/post-policy_test.go000066400000000000000000000251101510244170000171750ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2023 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "strings" "testing" "time" "github.com/minio/minio-go/v7/pkg/encrypt" ) func TestPostPolicySetExpires(t *testing.T) { tests := []struct { name string input time.Time wantErr bool wantResult string }{ { name: "valid time", input: time.Date(2023, time.March, 2, 15, 4, 5, 0, time.UTC), wantErr: false, wantResult: "2023-03-02T15:04:05", }, { name: "time before 1970", input: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { pp := NewPostPolicy() err := pp.SetExpires(tt.input) if (err != nil) != tt.wantErr { t.Errorf("%s: want error: %v, got: %v", tt.name, tt.wantErr, err) } if tt.wantResult != "" { result := pp.String() if !strings.Contains(result, tt.wantResult) { t.Errorf("%s: want result to contain: '%s', got: '%s'", tt.name, tt.wantResult, result) } } }) } } func TestPostPolicySetKey(t *testing.T) { tests := []struct { name string input string wantErr bool wantResult string }{ { name: "valid key", input: "my-object", wantResult: `"eq","$key","my-object"`, }, { name: "empty key", input: "", wantErr: true, }, { name: "key with spaces", input: " ", wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { pp := NewPostPolicy() err := pp.SetKey(tt.input) if (err != nil) != tt.wantErr { t.Errorf("%s: want error: %v, got: %v", tt.name, tt.wantErr, err) } if tt.wantResult != "" { result := pp.String() if !strings.Contains(result, tt.wantResult) { t.Errorf("%s: want result to contain: '%s', got: '%s'", tt.name, tt.wantResult, result) } } }) } } func TestPostPolicySetKeyStartsWith(t *testing.T) { tests := []struct { name string input string want string }{ { name: "valid key prefix", input: "my-prefix/", want: `["starts-with","$key","my-prefix/"]`, }, { name: "empty prefix (allow any key)", input: "", want: `["starts-with","$key",""]`, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { pp := NewPostPolicy() err := pp.SetKeyStartsWith(tt.input) if err != nil { t.Errorf("%s: want no error, got: %v", tt.name, err) } if tt.want != "" { result := pp.String() if !strings.Contains(result, tt.want) { t.Errorf("%s: want result to contain: '%s', got: '%s'", tt.name, tt.want, result) } } }) } } func TestPostPolicySetBucket(t *testing.T) { tests := []struct { name string input string wantErr bool wantResult string }{ { name: "valid bucket", input: "my-bucket", wantResult: `"eq","$bucket","my-bucket"`, }, { name: "empty bucket", input: "", wantErr: true, }, { name: "bucket with spaces", input: " ", wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { pp := NewPostPolicy() err := pp.SetBucket(tt.input) if (err != nil) != tt.wantErr { t.Errorf("%s: want error: %v, got: %v", tt.name, tt.wantErr, err) } if tt.wantResult != "" { result := pp.String() if !strings.Contains(result, tt.wantResult) { t.Errorf("%s: want result to contain: '%s', got: '%s'", tt.name, tt.wantResult, result) } } }) } } func TestPostPolicySetCondition(t *testing.T) { tests := []struct { name string matchType string condition string value string wantErr bool wantResult string }{ { name: "valid eq condition", matchType: "eq", condition: "X-Amz-Date", value: "20210324T000000Z", wantResult: `"eq","$X-Amz-Date","20210324T000000Z"`, }, { name: "empty value", matchType: "eq", condition: "X-Amz-Date", value: "", wantErr: true, }, { name: "invalid condition", matchType: "eq", condition: "Invalid-Condition", value: "somevalue", wantErr: true, }, { name: "valid starts-with condition", matchType: "starts-with", condition: "X-Amz-Credential", value: "my-access-key", wantResult: `"starts-with","$X-Amz-Credential","my-access-key"`, }, { name: "empty condition", matchType: "eq", condition: "", value: "somevalue", wantErr: true, }, { name: "empty matchType", matchType: "", condition: "X-Amz-Date", value: "somevalue", wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { pp := NewPostPolicy() err := pp.SetCondition(tt.matchType, tt.condition, tt.value) if (err != nil) != tt.wantErr { t.Errorf("%s: want error: %v, got: %v", tt.name, tt.wantErr, err) } if tt.wantResult != "" { result := pp.String() if !strings.Contains(result, tt.wantResult) { t.Errorf("%s: want result to contain: '%s', got: '%s'", tt.name, tt.wantResult, result) } } }) } } func TestPostPolicySetTagging(t *testing.T) { tests := []struct { name string tagging string wantErr bool wantResult string }{ { name: "valid tagging", tagging: `key1value1`, wantResult: `"eq","$tagging","key1value1"`, }, { name: "empty tagging", tagging: "", wantErr: true, }, { name: "whitespace tagging", tagging: " ", wantErr: true, }, { name: "invalid XML", tagging: `key1value1`, wantErr: true, }, { name: "invalid schema", tagging: ``, wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { pp := NewPostPolicy() err := pp.SetTagging(tt.tagging) if (err != nil) != tt.wantErr { t.Errorf("%s: want error: %v, got: %v", tt.name, tt.wantErr, err) } if tt.wantResult != "" { result := pp.String() if !strings.Contains(result, tt.wantResult) { t.Errorf("%s: want result to contain: '%s', got: '%s'", tt.name, tt.wantResult, result) } } }) } } func TestPostPolicySetUserMetadata(t *testing.T) { tests := []struct { name string key string value string wantErr bool wantResult string }{ { name: "valid metadata", key: "user-key", value: "user-value", wantResult: `"eq","$x-amz-meta-user-key","user-value"`, }, { name: "empty key", key: "", value: "somevalue", wantErr: true, }, { name: "empty value", key: "user-key", value: "", wantErr: true, }, { name: "key with spaces", key: " ", value: "somevalue", wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { pp := NewPostPolicy() err := pp.SetUserMetadata(tt.key, tt.value) if (err != nil) != tt.wantErr { t.Errorf("%s: want error: %v, got: %v", tt.name, tt.wantErr, err) } if tt.wantResult != "" { result := pp.String() if !strings.Contains(result, tt.wantResult) { t.Errorf("%s: want result to contain: '%s', got: '%s'", tt.name, tt.wantResult, result) } } }) } } func TestPostPolicySetChecksum(t *testing.T) { tests := []struct { name string checksum Checksum wantErr bool wantResult string }{ { name: "valid checksum SHA256", checksum: ChecksumSHA256.ChecksumBytes([]byte("somerandomdata")), wantResult: `[["eq","$x-amz-checksum-algorithm","SHA256"],["eq","$x-amz-checksum-sha256","29/7Qm/iMzZ1O3zMbO0luv6mYWyS6JIqPYV9lc8w1PA="]]`, }, { name: "valid checksum CRC32", checksum: ChecksumCRC32.ChecksumBytes([]byte("somerandomdata")), wantResult: `[["eq","$x-amz-checksum-algorithm","CRC32"],["eq","$x-amz-checksum-crc32","7sOPnw=="]]`, }, { name: "empty checksum", checksum: Checksum{}, wantResult: "", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { pp := NewPostPolicy() err := pp.SetChecksum(tt.checksum) if (err != nil) != tt.wantErr { t.Errorf("%s: want error: %v, got: %v", tt.name, tt.wantErr, err) } if tt.wantResult != "" { result := pp.String() if !strings.Contains(result, tt.wantResult) { t.Errorf("%s: want result to contain: '%s', got: '%s'", tt.name, tt.wantResult, result) } } }) } } func TestPostPolicySetEncryption(t *testing.T) { tests := []struct { name string sseType string keyID string want map[string]string }{ { name: "SSE-S3 encryption", sseType: "SSE-S3", keyID: "my-key-id", want: map[string]string{ "X-Amz-Server-Side-Encryption": "aws:kms", "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": "my-key-id", }, }, { name: "SSE-C encryption with Key ID", sseType: "SSE-C", keyID: "my-key-id", want: map[string]string{ "X-Amz-Server-Side-Encryption-Customer-Key": "bXktc2VjcmV0LWtleTEyMzQ1Njc4OTBhYmNkZWZnaGk=", "X-Amz-Server-Side-Encryption-Customer-Key-Md5": "T1mefJwyXBH43sRtfEgRZQ==", "X-Amz-Server-Side-Encryption-Customer-Algorithm": "AES256", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { pp := NewPostPolicy() var sse encrypt.ServerSide var err error switch tt.sseType { case "SSE-S3": sse, err = encrypt.NewSSEKMS(tt.keyID, nil) if err != nil { t.Fatalf("Failed to create SSE-KMS: %v", err) } case "SSE-C": sse, err = encrypt.NewSSEC([]byte("my-secret-key1234567890abcdefghi")) if err != nil { t.Fatalf("Failed to create SSE-C: %v", err) } default: t.Fatalf("Unknown SSE type: %s", tt.sseType) } pp.SetEncryption(sse) for k, v := range tt.want { if pp.formData[k] != v { t.Errorf("%s: want %s: %s, got: %s", tt.name, k, v, pp.formData[k]) } } }) } } minio-go-7.0.97/retry-continous.go000066400000000000000000000035151510244170000170450ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "iter" "math" "time" ) // newRetryTimerContinous creates a timer with exponentially increasing delays forever. func (c *Client) newRetryTimerContinous(baseSleep, maxSleep time.Duration, jitter float64) iter.Seq[int] { // normalize jitter to the range [0, 1.0] if jitter < NoJitter { jitter = NoJitter } if jitter > MaxJitter { jitter = MaxJitter } // computes the exponential backoff duration according to // https://www.awsarchitectureblog.com/2015/03/backoff.html exponentialBackoffWait := func(attempt int) time.Duration { // 1< maxAttempt { attempt = maxAttempt } // sleep = random_between(0, min(maxSleep, base * 2 ** attempt)) sleep := baseSleep * time.Duration(1< maxSleep { sleep = maxSleep } if math.Abs(jitter-NoJitter) > 1e-9 { sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter) } return sleep } return func(yield func(int) bool) { var nextBackoff int for { if !yield(nextBackoff) { return } nextBackoff++ time.Sleep(exponentialBackoffWait(nextBackoff)) } } } minio-go-7.0.97/retry.go000066400000000000000000000110461510244170000150240ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "crypto/x509" "errors" "iter" "math" "net/http" "net/url" "time" ) // MaxRetry is the maximum number of retries before stopping. var MaxRetry = 10 // MaxJitter will randomize over the full exponential backoff time const MaxJitter = 1.0 // NoJitter disables the use of jitter for randomizing the exponential backoff time const NoJitter = 0.0 // DefaultRetryUnit - default unit multiplicative per retry. // defaults to 200 * time.Millisecond var DefaultRetryUnit = 200 * time.Millisecond // DefaultRetryCap - Each retry attempt never waits no longer than // this maximum time duration. var DefaultRetryCap = time.Second // newRetryTimer creates a timer with exponentially increasing // delays until the maximum retry attempts are reached. func (c *Client) newRetryTimer(ctx context.Context, maxRetry int, baseSleep, maxSleep time.Duration, jitter float64) iter.Seq[int] { // computes the exponential backoff duration according to // https://www.awsarchitectureblog.com/2015/03/backoff.html exponentialBackoffWait := func(attempt int) time.Duration { // normalize jitter to the range [0, 1.0] if jitter < NoJitter { jitter = NoJitter } if jitter > MaxJitter { jitter = MaxJitter } // sleep = random_between(0, min(maxSleep, base * 2 ** attempt)) sleep := baseSleep * time.Duration(1< maxSleep { sleep = maxSleep } if math.Abs(jitter-NoJitter) > 1e-9 { sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter) } return sleep } return func(yield func(int) bool) { // if context is already canceled, skip yield select { case <-ctx.Done(): return default: } for i := range maxRetry { if !yield(i) { return } select { case <-time.After(exponentialBackoffWait(i)): case <-ctx.Done(): return } } } } // List of AWS S3 error codes which are retryable. var retryableS3Codes = map[string]struct{}{ "RequestError": {}, "RequestTimeout": {}, "Throttling": {}, "ThrottlingException": {}, "RequestLimitExceeded": {}, "RequestThrottled": {}, "InternalError": {}, "ExpiredToken": {}, "ExpiredTokenException": {}, "SlowDown": {}, "SlowDownWrite": {}, "SlowDownRead": {}, // Add more AWS S3 codes here. } // isS3CodeRetryable - is s3 error code retryable. func isS3CodeRetryable(s3Code string) (ok bool) { _, ok = retryableS3Codes[s3Code] return ok } // List of HTTP status codes which are retryable. var retryableHTTPStatusCodes = map[int]struct{}{ http.StatusRequestTimeout: {}, 429: {}, // http.StatusTooManyRequests is not part of the Go 1.5 library, yet 499: {}, // client closed request, retry. A non-standard status code introduced by nginx. http.StatusInternalServerError: {}, http.StatusBadGateway: {}, http.StatusServiceUnavailable: {}, http.StatusGatewayTimeout: {}, 520: {}, // It is used by Cloudflare as a catch-all response for when the origin server sends something unexpected. // Add more HTTP status codes here. } // isHTTPStatusRetryable - is HTTP error code retryable. func isHTTPStatusRetryable(httpStatusCode int) (ok bool) { _, ok = retryableHTTPStatusCodes[httpStatusCode] return ok } // For now, all http Do() requests are retriable except some well defined errors func isRequestErrorRetryable(ctx context.Context, err error) bool { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { // Retry if internal timeout in the HTTP call. return ctx.Err() == nil } if ue, ok := err.(*url.Error); ok { e := ue.Unwrap() switch e.(type) { // x509: certificate signed by unknown authority case x509.UnknownAuthorityError: return false } switch e.Error() { case "http: server gave HTTP response to HTTPS client": return false } } return true } minio-go-7.0.97/retry_test.go000066400000000000000000000066471510244170000160760ustar00rootroot00000000000000package minio import ( "context" "math/rand" "testing" "time" ) func TestRetryTimer(t *testing.T) { t.Run("withLimit", func(t *testing.T) { t.Parallel() c := &Client{random: rand.New(rand.NewSource(42))} ctx := context.Background() var count int for range c.newRetryTimer(ctx, 3, time.Millisecond, 10*time.Millisecond, 0.0) { count++ } if count != 3 { t.Errorf("expected exactly 3 yields") } }) t.Run("checkDelay", func(t *testing.T) { t.Parallel() c := &Client{random: rand.New(rand.NewSource(42))} ctx := context.Background() prev := time.Now() baseSleep := time.Millisecond maxSleep := 10 * time.Millisecond for i := range c.newRetryTimer(ctx, 6, baseSleep, maxSleep, 0.0) { if i == 0 { // there is no sleep for the first execution if time.Since(prev) >= time.Millisecond { t.Errorf("expected to not sleep for the first instance of the loop") } prev = time.Now() continue } expect := baseSleep * time.Duration(1< 2*maxSleep { t.Errorf("expected to sleep for at least %s", expect.String()) } prev = time.Now() } }) t.Run("withBreak", func(t *testing.T) { t.Parallel() c := &Client{random: rand.New(rand.NewSource(42))} ctx := context.Background() var count int for range c.newRetryTimer(ctx, 10, time.Millisecond, 10*time.Millisecond, 0.5) { count++ if count >= 3 { break } } if count != 3 { t.Errorf("expected exactly 3 yields") } }) t.Run("withCancelledContext", func(t *testing.T) { t.Parallel() c := &Client{random: rand.New(rand.NewSource(42))} ctx := context.Background() ctx, cancel := context.WithCancel(ctx) cancel() var count int for range c.newRetryTimer(ctx, 10, time.Millisecond, 10*time.Millisecond, 0.5) { count++ } if count != 0 { t.Errorf("expected no yields") } }) t.Run("whileCancelledContext", func(t *testing.T) { t.Parallel() c := &Client{random: rand.New(rand.NewSource(42))} ctx := context.Background() ctx, cancel := context.WithCancel(ctx) var count int for range c.newRetryTimer(ctx, 10, time.Millisecond, 10*time.Millisecond, 0.5) { count++ cancel() } cancel() if count != 1 { t.Errorf("expected only one yield") } }) } func TestRetryContinuous(t *testing.T) { t.Run("checkDelay", func(t *testing.T) { t.Parallel() c := &Client{random: rand.New(rand.NewSource(42))} prev := time.Now() baseSleep := time.Millisecond maxSleep := 10 * time.Millisecond for i := range c.newRetryTimerContinous(time.Millisecond, 10*time.Millisecond, 0.0) { if i == 0 { // there is no sleep for the first execution if time.Since(prev) >= time.Millisecond { t.Errorf("expected to not sleep for the first instance of the loop") } prev = time.Now() continue } expect := baseSleep * time.Duration(1< 2*maxSleep { t.Errorf("expected to sleep for at least %s", expect.String()) } prev = time.Now() if i >= 10 { break } } }) t.Run("withBreak", func(t *testing.T) { t.Parallel() c := &Client{random: rand.New(rand.NewSource(42))} var count int for range c.newRetryTimerContinous(time.Millisecond, 10*time.Millisecond, 0.5) { count++ if count >= 3 { break } } if count != 3 { t.Errorf("expected exactly 3 yields") } }) } minio-go-7.0.97/s3-error.go000066400000000000000000000165661510244170000153470ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio // Constants for error keys const ( NoSuchBucket = "NoSuchBucket" NoSuchKey = "NoSuchKey" NoSuchUpload = "NoSuchUpload" AccessDenied = "AccessDenied" Conflict = "Conflict" PreconditionFailed = "PreconditionFailed" InvalidArgument = "InvalidArgument" EntityTooLarge = "EntityTooLarge" EntityTooSmall = "EntityTooSmall" UnexpectedEOF = "UnexpectedEOF" APINotSupported = "APINotSupported" InvalidRegion = "InvalidRegion" NoSuchBucketPolicy = "NoSuchBucketPolicy" BadDigest = "BadDigest" IncompleteBody = "IncompleteBody" InternalError = "InternalError" InvalidAccessKeyID = "InvalidAccessKeyId" InvalidBucketName = "InvalidBucketName" InvalidDigest = "InvalidDigest" InvalidRange = "InvalidRange" MalformedXML = "MalformedXML" MissingContentLength = "MissingContentLength" MissingContentMD5 = "MissingContentMD5" MissingRequestBodyError = "MissingRequestBodyError" NotImplemented = "NotImplemented" RequestTimeTooSkewed = "RequestTimeTooSkewed" SignatureDoesNotMatch = "SignatureDoesNotMatch" MethodNotAllowed = "MethodNotAllowed" InvalidPart = "InvalidPart" InvalidPartOrder = "InvalidPartOrder" InvalidObjectState = "InvalidObjectState" AuthorizationHeaderMalformed = "AuthorizationHeaderMalformed" MalformedPOSTRequest = "MalformedPOSTRequest" BucketNotEmpty = "BucketNotEmpty" AllAccessDisabled = "AllAccessDisabled" MalformedPolicy = "MalformedPolicy" MissingFields = "MissingFields" AuthorizationQueryParametersError = "AuthorizationQueryParametersError" MalformedDate = "MalformedDate" BucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou" InvalidDuration = "InvalidDuration" XAmzContentSHA256Mismatch = "XAmzContentSHA256Mismatch" XMinioInvalidObjectName = "XMinioInvalidObjectName" NoSuchCORSConfiguration = "NoSuchCORSConfiguration" BucketAlreadyExists = "BucketAlreadyExists" NoSuchVersion = "NoSuchVersion" NoSuchTagSet = "NoSuchTagSet" Testing = "Testing" Success = "Success" ) // Non exhaustive list of AWS S3 standard error responses - // http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html var s3ErrorResponseMap = map[string]string{ AccessDenied: "Access Denied.", BadDigest: "The Content-Md5 you specified did not match what we received.", EntityTooSmall: "Your proposed upload is smaller than the minimum allowed object size.", EntityTooLarge: "Your proposed upload exceeds the maximum allowed object size.", IncompleteBody: "You did not provide the number of bytes specified by the Content-Length HTTP header.", InternalError: "We encountered an internal error, please try again.", InvalidAccessKeyID: "The access key ID you provided does not exist in our records.", InvalidBucketName: "The specified bucket is not valid.", InvalidDigest: "The Content-Md5 you specified is not valid.", InvalidRange: "The requested range is not satisfiable.", MalformedXML: "The XML you provided was not well-formed or did not validate against our published schema.", MissingContentLength: "You must provide the Content-Length HTTP header.", MissingContentMD5: "Missing required header for this request: Content-Md5.", MissingRequestBodyError: "Request body is empty.", NoSuchBucket: "The specified bucket does not exist.", NoSuchBucketPolicy: "The bucket policy does not exist.", NoSuchKey: "The specified key does not exist.", NoSuchUpload: "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.", NotImplemented: "A header you provided implies functionality that is not implemented.", PreconditionFailed: "At least one of the pre-conditions you specified did not hold.", RequestTimeTooSkewed: "The difference between the request time and the server's time is too large.", SignatureDoesNotMatch: "The request signature we calculated does not match the signature you provided. Check your key and signing method.", MethodNotAllowed: "The specified method is not allowed against this resource.", InvalidPart: "One or more of the specified parts could not be found.", InvalidPartOrder: "The list of parts was not in ascending order. The parts list must be specified in order by part number.", InvalidObjectState: "The operation is not valid for the current state of the object.", AuthorizationHeaderMalformed: "The authorization header is malformed; the region is wrong.", MalformedPOSTRequest: "The body of your POST request is not well-formed multipart/form-data.", BucketNotEmpty: "The bucket you tried to delete is not empty.", AllAccessDisabled: "All access to this bucket has been disabled.", MalformedPolicy: "Policy has invalid resource.", MissingFields: "Missing fields in request.", AuthorizationQueryParametersError: "Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting \"/YYYYMMDD/REGION/SERVICE/aws4_request\".", MalformedDate: "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.", BucketAlreadyOwnedByYou: "Your previous request to create the named bucket succeeded and you already own it.", InvalidDuration: "Duration provided in the request is invalid.", XAmzContentSHA256Mismatch: "The provided 'x-amz-content-sha256' header does not match what was computed.", NoSuchCORSConfiguration: "The specified bucket does not have a CORS configuration.", Conflict: "Bucket not empty.", // Add new API errors here. } minio-go-7.0.97/test-utils_test.go000066400000000000000000000041351510244170000170340ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "bytes" "encoding/xml" "io" "net/http" "strconv" ) // Contains common used utilities for tests. // APIError Used for mocking error response from server. type APIError struct { Code string Description string HTTPStatusCode int } // Mocks XML error response from the server. func generateErrorResponse(resp *http.Response, APIErr APIError, bucketName string) *http.Response { // generate error response. errorResponse := getAPIErrorResponse(APIErr, bucketName) encodedErrorResponse := encodeResponse(errorResponse) // write Header. resp.StatusCode = APIErr.HTTPStatusCode resp.Body = io.NopCloser(bytes.NewBuffer(encodedErrorResponse)) return resp } // getErrorResponse gets in standard error and resource value and // provides a encodable populated response values. func getAPIErrorResponse(err APIError, bucketName string) ErrorResponse { errResp := ErrorResponse{} errResp.Code = err.Code errResp.Message = err.Description errResp.BucketName = bucketName return errResp } // Encodes the response headers into XML format. func encodeResponse(response interface{}) []byte { var bytesBuffer bytes.Buffer bytesBuffer.WriteString(xml.Header) encode := xml.NewEncoder(&bytesBuffer) encode.Encode(response) return bytesBuffer.Bytes() } // Convert string to bool and always return false if any error func mustParseBool(str string) bool { b, err := strconv.ParseBool(str) if err != nil { return false } return b } minio-go-7.0.97/testcerts/000077500000000000000000000000001510244170000153465ustar00rootroot00000000000000minio-go-7.0.97/testcerts/private.key000066400000000000000000000032501510244170000175320ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCwUyKC2VOXy2+8 gMQkRrDJ4aA7K5pgj6LHWu25GeY93x+8DLFyQ9BhoaMcAbs2Cmw91rONDrZ0gNql yi5JX8t+iiVH8o6dcq6W8jNLnOw0GMNJ2/E1Ckfe5ktkn9synSSwMdnFp3cDk7Hb 2j6IiWrb+PXb7VGL47kDrG59iKQ350MiB3PNpd1ulHbi2m2ZC3WyoTTzlgeTXiXa zhBIX4wsGVYs6RzS1bTZFBq05dIPNMJCRDVBSBYAAVuBxKjh4xvhC6j0rTCCK8uJ 752KioW4Y0VAEv6yUC4Ht6D9Jcj7gODTgb2irWSCNXFH+pZaI6wWlS8pPiL6iljY P3kBeFiLAgMBAAECggEAKM20SM9+FryPSPILcdGiC7XY3JiEix/yLWwPYyxpKZw+ vce6MJUc3dsH4e1Mo37Z+Z17w4LKGj/PWVpmR7iRYOEbK4EoG6t0V54I3NCdoJiy aJ8rPHj6lMx6WfjcQuQ2n0eJ+8F7OyqsmBHzMqmKPwln69MJcfPq1rzKfOZoCj9p 0oZ+3Iv3roC4uH8peZFooCDUlzJL+8KiybVlemNfklKsHfRmL2vOdFBt+qvit6N/ 9JgBTX1mRx1+vqECj+TlVP//k3BTEPNfpIvsLCRN0eBbQcXYzu/gZfHwGnsy5Lxy HaHNJnmLZMWSCc4iyCK7uN/BHXNUSSh3qqp4wqz0IQKBgQDdGbOuOVdJW4J1yYua nDLAu2RQqvZTnoz1/b8jcrjS7tS1la5x1IN0Z9/VqTmkfxyHOK9ab1iVlklkIzjP CmHnadUwr8vrLcdicFpjVLQU3O4ZqGrgiSGIPAotvOfAOuuzMs+r5ElW/MrGq0Pa /3tGCTIx8JscZZjGhffUNoIGeQKBgQDMKB+flQB9Ajeo1JM4y3DtHbMJ5D2+/qoe IkM7kN5K85EEpNwA2PMNKL2qthgM9YFU3K6Dj0gxPNsUKg3W7Ff2r+gaj8K+VjU0 VbdhTZANbou8hU551swDUCUgquassMtZJIdZnQ7puwLGK67sZwWlOS6Pe1aqaNc5 nY/MRbemIwKBgEySfykCkNlGCPuUDnZATE91VrudSewRyA3VkGHNdHcQ4bf1m9Gu YMxqwRl1HxJ6Nz4ZgplWYJ6FyusUS7NgjCGiBIR1DbFoTFoqQROPnUJwdUGLk2Ap /eP5ryjB+J0ZitGn8kY8rK2kpPGDFN/+hQnvW2PySTXfdbajZP4o1oU5AoGAMiT0 x3yQlyPRSf2Uf5Gwlf0Ceb5+0Ae6/xXJT7sgbmZuyyY3B1pCMIw+MczyEVTHxHFD x/qMb9OTt9swdQauAGBqcQO4gImqHcWj+hlT9Yied9qCUPjKOVIZHHH9oJL4D1gi iodCH3SYlNYr69LOFyv5XLKdsdN4caVaqYDCP+MCgYEAwXyCmSml5oCxJeAOrEDC Yg3vq3Ul9JO1wc8VDXn9+DtnFsuRHm0fTIxBelmis8AjIIq5DcObpk6wGYZwUiTU LYQU7v0/Azujv9cl10GI8wzYKiRvExZDTn0sp6OKnau735qBUZvsRDqEQQ5n7waZ xjlGmZyfah17laYZV9aJoHk= -----END PRIVATE KEY----- minio-go-7.0.97/testcerts/public.crt000066400000000000000000000027411510244170000173420ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIEKjCCApKgAwIBAgIRAPVKnAiFmDti207oQPs2VfUwDQYJKoZIhvcNAQELBQAw VTEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMRUwEwYDVQQLDAxoYXJz aGFAcHJ0c2MxHDAaBgNVBAMME21rY2VydCBoYXJzaGFAcHJ0c2MwHhcNMTkwMTA3 MTE1ODE2WhcNMjkwMTA3MTE1ODE2WjBEMScwJQYDVQQKEx5ta2NlcnQgZGV2ZWxv cG1lbnQgY2VydGlmaWNhdGUxGTAXBgNVBAsMEGhhcnNoYUBiYWNrc3BhY2UwggEi MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCwUyKC2VOXy2+8gMQkRrDJ4aA7 K5pgj6LHWu25GeY93x+8DLFyQ9BhoaMcAbs2Cmw91rONDrZ0gNqlyi5JX8t+iiVH 8o6dcq6W8jNLnOw0GMNJ2/E1Ckfe5ktkn9synSSwMdnFp3cDk7Hb2j6IiWrb+PXb 7VGL47kDrG59iKQ350MiB3PNpd1ulHbi2m2ZC3WyoTTzlgeTXiXazhBIX4wsGVYs 6RzS1bTZFBq05dIPNMJCRDVBSBYAAVuBxKjh4xvhC6j0rTCCK8uJ752KioW4Y0VA Ev6yUC4Ht6D9Jcj7gODTgb2irWSCNXFH+pZaI6wWlS8pPiL6iljYP3kBeFiLAgMB AAGjgYUwgYIwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwG A1UdEwEB/wQCMAAwHwYDVR0jBBgwFoAUD575sRLoRt9dCxSRqbVctoEHt3MwLAYD VR0RBCUwI4IJbG9jYWxob3N0hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMA0GCSqG SIb3DQEBCwUAA4IBgQC7qDRDNAHtfGtQs1UmvqWvHPI7qcBQgAibYq/Fox6X9ia1 weQBfNWEoNOsk97wzbTz81ifXIQ0oV11kWE8EdsbXOf9xeFe9FmDn10d4bGjuMLd +N3OtGKxLWry2xDYEsVHJZxVxwrf5GK6AJSJj/S837Nil6uRuwjvBVTbxmh1q0nV x63V8Ag65rLS0fu8msSb64N5UHMCQk6IE+BFHY2gh0lBfZHMdtP4IbeCm756K78/ WMeqjavGA3bqzVTixCHnJ9S2VLk/oQUS6mL869jM8+tN5VeE6Qsr1/Q5h+NaFCJg Ed5xjT9mmnc3BLsOHflb1dg+rA90Zz9wphgebXbJhRNuuDRv81dtRPTzM+evGRGM iRKtiDpog+K0HulfX2g4ZQ1dItEjYz+JYgUFJG+yCvBlNZ/WsTrIVcUCFKaG5rUC aNqvKrSXfbzKQx7V/TtUAeSfRk7TBRn5qh8Pl+MmQQsB0L9hwTdnqTNn057tghu4 3/yIIBpzdWPhQ5uv7Vc= -----END CERTIFICATE----- minio-go-7.0.97/transport.go000066400000000000000000000046531510244170000157210ustar00rootroot00000000000000//go:build go1.7 || go1.8 // +build go1.7 go1.8 /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2017-2018 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "crypto/tls" "crypto/x509" "net" "net/http" "os" "time" ) // mustGetSystemCertPool - return system CAs or empty pool in case of error (or windows) func mustGetSystemCertPool() *x509.CertPool { pool, err := x509.SystemCertPool() if err != nil { return x509.NewCertPool() } return pool } // DefaultTransport - this default transport is similar to // http.DefaultTransport but with additional param DisableCompression // is set to true to avoid decompressing content with 'gzip' encoding. var DefaultTransport = func(secure bool) (*http.Transport, error) { tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).DialContext, MaxIdleConns: 256, MaxIdleConnsPerHost: 16, ResponseHeaderTimeout: time.Minute, IdleConnTimeout: time.Minute, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 10 * time.Second, // Set this value so that the underlying transport round-tripper // doesn't try to auto decode the body of objects with // content-encoding set to `gzip`. // // Refer: // https://golang.org/src/net/http/transport.go?h=roundTrip#L1843 DisableCompression: true, } if secure { tr.TLSClientConfig = &tls.Config{ // Can't use SSLv3 because of POODLE and BEAST // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher // Can't use TLSv1.1 because of RC4 cipher usage MinVersion: tls.VersionTLS12, } if f := os.Getenv("SSL_CERT_FILE"); f != "" { rootCAs := mustGetSystemCertPool() data, err := os.ReadFile(f) if err == nil { rootCAs.AppendCertsFromPEM(data) } tr.TLSClientConfig.RootCAs = rootCAs } } return tr, nil } minio-go-7.0.97/utils.go000066400000000000000000000616121510244170000150230ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "context" "crypto/md5" "crypto/sha256" "crypto/tls" "encoding/base64" "encoding/hex" "encoding/xml" "errors" "fmt" "hash" "io" "math/rand" "mime" "net" "net/http" "net/url" "regexp" "strconv" "strings" "sync" "time" md5simd "github.com/minio/md5-simd" "github.com/minio/minio-go/v7/pkg/s3utils" "github.com/minio/minio-go/v7/pkg/tags" ) func trimEtag(etag string) string { etag = strings.TrimPrefix(etag, "\"") return strings.TrimSuffix(etag, "\"") } var expirationRegex = regexp.MustCompile(`expiry-date="(.*?)", rule-id="(.*?)"`) func amzExpirationToExpiryDateRuleID(expiration string) (time.Time, string) { if matches := expirationRegex.FindStringSubmatch(expiration); len(matches) == 3 { expTime, err := parseRFC7231Time(matches[1]) if err != nil { return time.Time{}, "" } return expTime, matches[2] } return time.Time{}, "" } var restoreRegex = regexp.MustCompile(`ongoing-request="(.*?)"(, expiry-date="(.*?)")?`) func amzRestoreToStruct(restore string) (ongoing bool, expTime time.Time, err error) { matches := restoreRegex.FindStringSubmatch(restore) if len(matches) != 4 { return false, time.Time{}, errors.New("unexpected restore header") } ongoing, err = strconv.ParseBool(matches[1]) if err != nil { return false, time.Time{}, err } if matches[3] != "" { expTime, err = parseRFC7231Time(matches[3]) if err != nil { return false, time.Time{}, err } } return ongoing, expTime, err } // xmlDecoder provide decoded value in xml. func xmlDecoder(body io.Reader, v interface{}) error { d := xml.NewDecoder(body) return d.Decode(v) } // sum256 calculate sha256sum for an input byte array, returns hex encoded. func sum256Hex(data []byte) string { hash := newSHA256Hasher() defer hash.Close() hash.Write(data) return hex.EncodeToString(hash.Sum(nil)) } // sumMD5Base64 calculate md5sum for an input byte array, returns base64 encoded. func sumMD5Base64(data []byte) string { hash := newMd5Hasher() defer hash.Close() hash.Write(data) return base64.StdEncoding.EncodeToString(hash.Sum(nil)) } // getEndpointURL - construct a new endpoint. func getEndpointURL(endpoint string, secure bool) (*url.URL, error) { // If secure is false, use 'http' scheme. scheme := "https" if !secure { scheme = "http" } // Construct a secured endpoint URL. endpointURLStr := scheme + "://" + endpoint endpointURL, err := url.Parse(endpointURLStr) if err != nil { return nil, err } // Validate incoming endpoint URL. if err := isValidEndpointURL(*endpointURL); err != nil { return nil, err } return endpointURL, nil } // closeResponse close non nil response with any response Body. // convenient wrapper to drain any remaining data on response body. // // Subsequently this allows golang http RoundTripper // to re-use the same connection for future requests. func closeResponse(resp *http.Response) { // Callers should close resp.Body when done reading from it. // If resp.Body is not closed, the Client's underlying RoundTripper // (typically Transport) may not be able to re-use a persistent TCP // connection to the server for a subsequent "keep-alive" request. if resp != nil && resp.Body != nil { // Drain any remaining Body and then close the connection. // Without this closing connection would disallow re-using // the same connection for future uses. // - http://stackoverflow.com/a/17961593/4465767 io.Copy(io.Discard, resp.Body) resp.Body.Close() } } var ( // Hex encoded string of nil sha256sum bytes. emptySHA256Hex = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" // Sentinel URL is the default url value which is invalid. sentinelURL = url.URL{} ) // Verify if input endpoint URL is valid. func isValidEndpointURL(endpointURL url.URL) error { if endpointURL == sentinelURL { return errInvalidArgument("Endpoint url cannot be empty.") } if endpointURL.Path != "/" && endpointURL.Path != "" { return errInvalidArgument("Endpoint url cannot have fully qualified paths.") } host := endpointURL.Hostname() if !s3utils.IsValidIP(host) && !s3utils.IsValidDomain(host) { msg := "Endpoint: " + endpointURL.Host + " does not follow ip address or domain name standards." return errInvalidArgument(msg) } if strings.Contains(host, ".s3.amazonaws.com") { if !s3utils.IsAmazonEndpoint(endpointURL) { return errInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'.") } } if strings.Contains(host, ".googleapis.com") { if !s3utils.IsGoogleEndpoint(endpointURL) { return errInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.") } } return nil } // Verify if input expires value is valid. func isValidExpiry(expires time.Duration) error { expireSeconds := int64(expires / time.Second) if expireSeconds < 1 { return errInvalidArgument("Expires cannot be lesser than 1 second.") } if expireSeconds > 604800 { return errInvalidArgument("Expires cannot be greater than 7 days.") } return nil } // Extract only necessary metadata header key/values by // filtering them out with a list of custom header keys. func extractObjMetadata(header http.Header) http.Header { preserveKeys := []string{ "Content-Type", "Cache-Control", "Content-Encoding", "Content-Language", "Content-Disposition", "X-Amz-Storage-Class", "X-Amz-Object-Lock-Mode", "X-Amz-Object-Lock-Retain-Until-Date", "X-Amz-Object-Lock-Legal-Hold", "X-Amz-Website-Redirect-Location", "X-Amz-Server-Side-Encryption", "X-Amz-Tagging-Count", "X-Amz-Meta-", "X-Minio-Meta-", // Add new headers to be preserved. // if you add new headers here, please extend // PutObjectOptions{} to preserve them // upon upload as well. } filteredHeader := make(http.Header) for k, v := range header { var found bool for _, prefix := range preserveKeys { if !strings.HasPrefix(k, prefix) { continue } found = true if prefix == "X-Amz-Meta-" || prefix == "X-Minio-Meta-" { for index, val := range v { if strings.HasPrefix(val, "=?") { decoder := mime.WordDecoder{} if decoded, err := decoder.DecodeHeader(val); err == nil { v[index] = decoded } } } } break } if found { filteredHeader[k] = v } } return filteredHeader } const ( // RFC 7231#section-7.1.1.1 timetamp format. e.g Tue, 29 Apr 2014 18:30:38 GMT rfc822TimeFormat = "Mon, 2 Jan 2006 15:04:05 GMT" rfc822TimeFormatSingleDigitDay = "Mon, _2 Jan 2006 15:04:05 GMT" rfc822TimeFormatSingleDigitDayTwoDigitYear = "Mon, _2 Jan 06 15:04:05 GMT" ) func parseTime(t string, formats ...string) (time.Time, error) { for _, format := range formats { tt, err := time.Parse(format, t) if err == nil { return tt, nil } } return time.Time{}, fmt.Errorf("unable to parse %s in any of the input formats: %s", t, formats) } func parseRFC7231Time(lastModified string) (time.Time, error) { return parseTime(lastModified, rfc822TimeFormat, rfc822TimeFormatSingleDigitDay, rfc822TimeFormatSingleDigitDayTwoDigitYear) } // ToObjectInfo converts http header values into ObjectInfo type, // extracts metadata and fills in all the necessary fields in ObjectInfo. func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, error) { var err error // Trim off the odd double quotes from ETag in the beginning and end. etag := trimEtag(h.Get("ETag")) // Parse content length is exists var size int64 = -1 contentLengthStr := h.Get("Content-Length") if contentLengthStr != "" { size, err = strconv.ParseInt(contentLengthStr, 10, 64) if err != nil { // Content-Length is not valid return ObjectInfo{}, ErrorResponse{ Code: InternalError, Message: fmt.Sprintf("Content-Length is not an integer, failed with %v", err), BucketName: bucketName, Key: objectName, RequestID: h.Get("x-amz-request-id"), HostID: h.Get("x-amz-id-2"), Region: h.Get("x-amz-bucket-region"), } } } // Parse Last-Modified has http time format. mtime, err := parseRFC7231Time(h.Get("Last-Modified")) if err != nil { return ObjectInfo{}, ErrorResponse{ Code: InternalError, Message: fmt.Sprintf("Last-Modified time format is invalid, failed with %v", err), BucketName: bucketName, Key: objectName, RequestID: h.Get("x-amz-request-id"), HostID: h.Get("x-amz-id-2"), Region: h.Get("x-amz-bucket-region"), } } // Fetch content type if any present. contentType := strings.TrimSpace(h.Get("Content-Type")) if contentType == "" { contentType = "application/octet-stream" } expiryStr := h.Get("Expires") var expiry time.Time if expiryStr != "" { expiry, err = parseRFC7231Time(expiryStr) if err != nil { return ObjectInfo{}, ErrorResponse{ Code: InternalError, Message: fmt.Sprintf("'Expiry' is not in supported format: %v", err), BucketName: bucketName, Key: objectName, RequestID: h.Get("x-amz-request-id"), HostID: h.Get("x-amz-id-2"), Region: h.Get("x-amz-bucket-region"), } } } metadata := extractObjMetadata(h) userMetadata := make(map[string]string) for k, v := range metadata { if strings.HasPrefix(k, "X-Amz-Meta-") { userMetadata[strings.TrimPrefix(k, "X-Amz-Meta-")] = v[0] } } userTags, err := tags.ParseObjectTags(h.Get(amzTaggingHeader)) if err != nil { return ObjectInfo{}, ErrorResponse{ Code: InternalError, } } var tagCount int if count := h.Get(amzTaggingCount); count != "" { tagCount, err = strconv.Atoi(count) if err != nil { return ObjectInfo{}, ErrorResponse{ Code: InternalError, Message: fmt.Sprintf("x-amz-tagging-count is not an integer, failed with %v", err), BucketName: bucketName, Key: objectName, RequestID: h.Get("x-amz-request-id"), HostID: h.Get("x-amz-id-2"), Region: h.Get("x-amz-bucket-region"), } } } // Nil if not found var restore *RestoreInfo if restoreHdr := h.Get(amzRestore); restoreHdr != "" { ongoing, expTime, err := amzRestoreToStruct(restoreHdr) if err != nil { return ObjectInfo{}, err } restore = &RestoreInfo{OngoingRestore: ongoing, ExpiryTime: expTime} } // extract lifecycle expiry date and rule ID expTime, ruleID := amzExpirationToExpiryDateRuleID(h.Get(amzExpiration)) deleteMarker := h.Get(amzDeleteMarker) == "true" // Save object metadata info. return ObjectInfo{ ETag: etag, Key: objectName, Size: size, LastModified: mtime, ContentType: contentType, Expires: expiry, VersionID: h.Get(amzVersionID), IsDeleteMarker: deleteMarker, ReplicationStatus: h.Get(amzReplicationStatus), Expiration: expTime, ExpirationRuleID: ruleID, // Extract only the relevant header keys describing the object. // following function filters out a list of standard set of keys // which are not part of object metadata. Metadata: metadata, UserMetadata: userMetadata, UserTags: userTags.ToMap(), UserTagCount: tagCount, Restore: restore, // Checksum values ChecksumCRC32: h.Get(ChecksumCRC32.Key()), ChecksumCRC32C: h.Get(ChecksumCRC32C.Key()), ChecksumSHA1: h.Get(ChecksumSHA1.Key()), ChecksumSHA256: h.Get(ChecksumSHA256.Key()), ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()), ChecksumMode: h.Get(ChecksumFullObjectMode.Key()), }, nil } var readFull = func(r io.Reader, buf []byte) (n int, err error) { // ReadFull reads exactly len(buf) bytes from r into buf. // It returns the number of bytes copied and an error if // fewer bytes were read. The error is EOF only if no bytes // were read. If an EOF happens after reading some but not // all the bytes, ReadFull returns ErrUnexpectedEOF. // On return, n == len(buf) if and only if err == nil. // If r returns an error having read at least len(buf) bytes, // the error is dropped. for n < len(buf) && err == nil { var nn int nn, err = r.Read(buf[n:]) // Some spurious io.Reader's return // io.ErrUnexpectedEOF when nn == 0 // this behavior is undocumented // so we are on purpose not using io.ReadFull // implementation because this can lead // to custom handling, to avoid that // we simply modify the original io.ReadFull // implementation to avoid this issue. // io.ErrUnexpectedEOF with nn == 0 really // means that io.EOF if err == io.ErrUnexpectedEOF && nn == 0 { err = io.EOF } n += nn } if n >= len(buf) { err = nil } else if n > 0 && err == io.EOF { err = io.ErrUnexpectedEOF } return n, err } // regCred matches credential string in HTTP header var regCred = regexp.MustCompile("Credential=([A-Z0-9]+)/") // regCred matches signature string in HTTP header var regSign = regexp.MustCompile("Signature=([[0-9a-f]+)") // Redact out signature value from authorization string. func redactSignature(origAuth string) string { if !strings.HasPrefix(origAuth, signV4Algorithm) { // Set a temporary redacted auth return "AWS **REDACTED**:**REDACTED**" } // Signature V4 authorization header. // Strip out accessKeyID from: // Credential=////aws4_request newAuth := regCred.ReplaceAllString(origAuth, "Credential=**REDACTED**/") // Strip out 256-bit signature from: Signature=<256-bit signature> return regSign.ReplaceAllString(newAuth, "Signature=**REDACTED**") } // Get default location returns the location based on the input // URL `u`, if region override is provided then all location // defaults to regionOverride. // // If no other cases match then the location is set to `us-east-1` // as a last resort. func getDefaultLocation(u url.URL, regionOverride string) (location string) { if regionOverride != "" { return regionOverride } region := s3utils.GetRegionFromURL(u) if region == "" { region = "us-east-1" } return region } var supportedHeaders = map[string]bool{ "content-type": true, "cache-control": true, "content-encoding": true, "content-disposition": true, "content-language": true, "x-amz-website-redirect-location": true, "x-amz-object-lock-mode": true, "x-amz-metadata-directive": true, "x-amz-object-lock-retain-until-date": true, "expires": true, "x-amz-replication-status": true, // Add more supported headers here. // Must be lower case. } // isStorageClassHeader returns true if the header is a supported storage class header func isStorageClassHeader(headerKey string) bool { return strings.EqualFold(amzStorageClass, headerKey) } // isStandardHeader returns true if header is a supported header and not a custom header func isStandardHeader(headerKey string) bool { return supportedHeaders[strings.ToLower(headerKey)] } // sseHeaders is list of server side encryption headers var sseHeaders = map[string]bool{ "x-amz-server-side-encryption": true, "x-amz-server-side-encryption-aws-kms-key-id": true, "x-amz-server-side-encryption-context": true, "x-amz-server-side-encryption-customer-algorithm": true, "x-amz-server-side-encryption-customer-key": true, "x-amz-server-side-encryption-customer-key-md5": true, // Add more supported headers here. // Must be lower case. } // isSSEHeader returns true if header is a server side encryption header. func isSSEHeader(headerKey string) bool { return sseHeaders[strings.ToLower(headerKey)] } // isAmzHeader returns true if header is a x-amz-meta-* or x-amz-acl header. func isAmzHeader(headerKey string) bool { key := strings.ToLower(headerKey) return strings.HasPrefix(key, "x-amz-meta-") || strings.HasPrefix(key, "x-amz-grant-") || key == "x-amz-acl" || isSSEHeader(headerKey) || strings.HasPrefix(key, "x-amz-checksum-") } // isMinioHeader returns true if header is x-minio- header. func isMinioHeader(headerKey string) bool { return strings.HasPrefix(strings.ToLower(headerKey), "x-minio-") } // supportedQueryValues is a list of query strings that can be passed in when using GetObject. var supportedQueryValues = map[string]bool{ "attributes": true, "partNumber": true, "versionId": true, "response-cache-control": true, "response-content-disposition": true, "response-content-encoding": true, "response-content-language": true, "response-content-type": true, "response-expires": true, } // isStandardQueryValue will return true when the passed in query string parameter is supported rather than customized. func isStandardQueryValue(qsKey string) bool { return supportedQueryValues[qsKey] } // Per documentation at https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html#LogFormatCustom, the // set of query params starting with "x-" are ignored by S3. const allowedCustomQueryPrefix = "x-" func isCustomQueryValue(qsKey string) bool { return strings.HasPrefix(qsKey, allowedCustomQueryPrefix) } var ( md5Pool = sync.Pool{New: func() interface{} { return md5.New() }} sha256Pool = sync.Pool{New: func() interface{} { return sha256.New() }} ) func newMd5Hasher() md5simd.Hasher { return &hashWrapper{Hash: md5Pool.Get().(hash.Hash), isMD5: true} } func newSHA256Hasher() md5simd.Hasher { return &hashWrapper{Hash: sha256Pool.Get().(hash.Hash), isSHA256: true} } // hashWrapper implements the md5simd.Hasher interface. type hashWrapper struct { hash.Hash isMD5 bool isSHA256 bool } // Close will put the hasher back into the pool. func (m *hashWrapper) Close() { if m.isMD5 && m.Hash != nil { m.Reset() md5Pool.Put(m.Hash) } if m.isSHA256 && m.Hash != nil { m.Reset() sha256Pool.Put(m.Hash) } m.Hash = nil } const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" const ( letterIdxBits = 6 // 6 bits to represent a letter index letterIdxMask = 1<= 0; { if remain == 0 { cache, remain = src.Int63(), letterIdxMax } if idx := int(cache & letterIdxMask); idx < len(letterBytes) { b[i] = letterBytes[idx] i-- } cache >>= letterIdxBits remain-- } return prefix + string(b[0:30-len(prefix)]) } // IsNetworkOrHostDown - if there was a network error or if the host is down. // expectTimeouts indicates that *context* timeouts are expected and does not // indicate a downed host. Other timeouts still returns down. func IsNetworkOrHostDown(err error, expectTimeouts bool) bool { if err == nil { return false } if errors.Is(err, context.Canceled) { return false } if expectTimeouts && errors.Is(err, context.DeadlineExceeded) { return false } if errors.Is(err, context.DeadlineExceeded) { return true } // We need to figure if the error either a timeout // or a non-temporary error. urlErr := &url.Error{} if errors.As(err, &urlErr) { switch urlErr.Err.(type) { case *net.DNSError, *net.OpError, net.UnknownNetworkError, *tls.CertificateVerificationError: return true } } var e net.Error if errors.As(err, &e) { if e.Timeout() { return true } } // Fallback to other mechanisms. switch { case strings.Contains(err.Error(), "Connection closed by foreign host"): return true case strings.Contains(err.Error(), "TLS handshake timeout"): // If error is - tlsHandshakeTimeoutError. return true case strings.Contains(err.Error(), "i/o timeout"): // If error is - tcp timeoutError. return true case strings.Contains(err.Error(), "connection timed out"): // If err is a net.Dial timeout. return true case strings.Contains(err.Error(), "connection refused"): // If err is connection refused return true case strings.Contains(err.Error(), "server gave HTTP response to HTTPS client"): // If err is TLS client is used with HTTP server return true case strings.Contains(err.Error(), "Client sent an HTTP request to an HTTPS server"): // If err is plain-text Client is used with a HTTPS server return true case strings.Contains(strings.ToLower(err.Error()), "503 service unavailable"): // Denial errors return true } return false } // newHashReaderWrapper will hash all reads done through r. // When r returns io.EOF the done function will be called with the sum. func newHashReaderWrapper(r io.Reader, h hash.Hash, done func(hash []byte)) *hashReaderWrapper { return &hashReaderWrapper{ r: r, h: h, done: done, } } type hashReaderWrapper struct { r io.Reader h hash.Hash done func(hash []byte) } // Read implements the io.Reader interface. func (h *hashReaderWrapper) Read(p []byte) (n int, err error) { n, err = h.r.Read(p) if n > 0 { n2, err := h.h.Write(p[:n]) if err != nil { return 0, err } if n2 != n { return 0, io.ErrShortWrite } } if err == io.EOF { // Call back h.done(h.h.Sum(nil)) } return n, err } // Following is ported from C to Go in 2016 by Justin Ruggles, with minimal alteration. // Used uint for unsigned long. Used uint32 for input arguments in order to match // the Go hash/crc32 package. zlib CRC32 combine (https://github.com/madler/zlib) // Modified for hash/crc64 by Klaus Post, 2024. func gf2MatrixTimes(mat []uint64, vec uint64) uint64 { var sum uint64 for vec != 0 { if vec&1 != 0 { sum ^= mat[0] } vec >>= 1 mat = mat[1:] } return sum } func gf2MatrixSquare(square, mat []uint64) { if len(square) != len(mat) { panic("square matrix size mismatch") } for n := range mat { square[n] = gf2MatrixTimes(mat, mat[n]) } } // crc32Combine returns the combined CRC-32 hash value of the two passed CRC-32 // hash values crc1 and crc2. poly represents the generator polynomial // and len2 specifies the byte length that the crc2 hash covers. func crc32Combine(poly uint32, crc1, crc2 uint32, len2 int64) uint32 { // degenerate case (also disallow negative lengths) if len2 <= 0 { return crc1 } even := make([]uint64, 32) // even-power-of-two zeros operator odd := make([]uint64, 32) // odd-power-of-two zeros operator // put operator for one zero bit in odd odd[0] = uint64(poly) // CRC-32 polynomial row := uint64(1) for n := 1; n < 32; n++ { odd[n] = row row <<= 1 } // put operator for two zero bits in even gf2MatrixSquare(even, odd) // put operator for four zero bits in odd gf2MatrixSquare(odd, even) // apply len2 zeros to crc1 (first square will put the operator for one // zero byte, eight zero bits, in even) crc1n := uint64(crc1) for { // apply zeros operator for this bit of len2 gf2MatrixSquare(even, odd) if len2&1 != 0 { crc1n = gf2MatrixTimes(even, crc1n) } len2 >>= 1 // if no more bits set, then done if len2 == 0 { break } // another iteration of the loop with odd and even swapped gf2MatrixSquare(odd, even) if len2&1 != 0 { crc1n = gf2MatrixTimes(odd, crc1n) } len2 >>= 1 // if no more bits set, then done if len2 == 0 { break } } // return combined crc crc1n ^= uint64(crc2) return uint32(crc1n) } func crc64Combine(poly uint64, crc1, crc2 uint64, len2 int64) uint64 { // degenerate case (also disallow negative lengths) if len2 <= 0 { return crc1 } even := make([]uint64, 64) // even-power-of-two zeros operator odd := make([]uint64, 64) // odd-power-of-two zeros operator // put operator for one zero bit in odd odd[0] = poly // CRC-64 polynomial row := uint64(1) for n := 1; n < 64; n++ { odd[n] = row row <<= 1 } // put operator for two zero bits in even gf2MatrixSquare(even, odd) // put operator for four zero bits in odd gf2MatrixSquare(odd, even) // apply len2 zeros to crc1 (first square will put the operator for one // zero byte, eight zero bits, in even) crc1n := crc1 for { // apply zeros operator for this bit of len2 gf2MatrixSquare(even, odd) if len2&1 != 0 { crc1n = gf2MatrixTimes(even, crc1n) } len2 >>= 1 // if no more bits set, then done if len2 == 0 { break } // another iteration of the loop with odd and even swapped gf2MatrixSquare(odd, even) if len2&1 != 0 { crc1n = gf2MatrixTimes(odd, crc1n) } len2 >>= 1 // if no more bits set, then done if len2 == 0 { break } } // return combined crc crc1n ^= crc2 return crc1n } minio-go-7.0.97/utils_test.go000066400000000000000000000443711510244170000160650ustar00rootroot00000000000000/* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2017 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package minio import ( "errors" "fmt" "math/rand" "mime" "net/http" "net/url" "reflect" "strings" "testing" "time" "github.com/minio/minio-go/v7/pkg/s3utils" ) func TestParseRFC7231Time(t *testing.T) { testCases := []struct { timeStr string expectedSuccess bool }{ { timeStr: "Sun, 2 Jan 2000 20:34:56 GMT", expectedSuccess: true, }, { timeStr: "Sun, 02 Jan 2000 20:34:56 GMT", expectedSuccess: true, }, { timeStr: "Sun, 2 Jan 00 20:34:56 GMT", expectedSuccess: true, }, { timeStr: "Sun, 02 Jan 00 20:34:56 GMT", expectedSuccess: true, }, { timeStr: "Su, 2 Jan 00 20:34:56 GMT", expectedSuccess: false, }, } for _, testCase := range testCases { testCase := testCase t.Run("", func(t *testing.T) { _, err := parseRFC7231Time(testCase.timeStr) if err != nil && testCase.expectedSuccess { t.Errorf("expected success found failure %v", err) } if err == nil && !testCase.expectedSuccess { t.Errorf("expected failure found success") } }) } } // Tests signature redacting function used // in filtering on-wire Authorization header. func TestRedactSignature(t *testing.T) { testCases := []struct { authValue string expectedRedactedAuthValue string }{ { authValue: "AWS 1231313:888x000231==", expectedRedactedAuthValue: "AWS **REDACTED**:**REDACTED**", }, { authValue: "AWS4-HMAC-SHA256 Credential=12312313/20170613/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=02131231312313213", expectedRedactedAuthValue: "AWS4-HMAC-SHA256 Credential=**REDACTED**/20170613/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=**REDACTED**", }, } for i, testCase := range testCases { redactedAuthValue := redactSignature(testCase.authValue) if redactedAuthValue != testCase.expectedRedactedAuthValue { t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedRedactedAuthValue, redactedAuthValue) } } } // Tests for 'getEndpointURL(endpoint string, inSecure bool)'. func TestGetEndpointURL(t *testing.T) { testCases := []struct { // Inputs. endPoint string secure bool // Expected result. result string err error // Flag indicating whether the test is expected to pass or not. shouldPass bool }{ {"s3.amazonaws.com", true, "https://s3.amazonaws.com", nil, true}, {"s3.cn-north-1.amazonaws.com.cn", true, "https://s3.cn-north-1.amazonaws.com.cn", nil, true}, {"s3.cn-northwest-1.amazonaws.com.cn", true, "https://s3.cn-northwest-1.amazonaws.com.cn", nil, true}, {"s3.amazonaws.com", false, "http://s3.amazonaws.com", nil, true}, {"s3.cn-north-1.amazonaws.com.cn", false, "http://s3.cn-north-1.amazonaws.com.cn", nil, true}, {"s3.cn-northwest-1.amazonaws.com.cn", false, "http://s3.cn-northwest-1.amazonaws.com.cn", nil, true}, {"192.168.1.1:9000", false, "http://192.168.1.1:9000", nil, true}, {"192.168.1.1:9000", true, "https://192.168.1.1:9000", nil, true}, {"s3.amazonaws.com:443", true, "https://s3.amazonaws.com:443", nil, true}, {"storage.googleapis.com:443", true, "https://storage.googleapis.com:443", nil, true}, {"[::1]", false, "http://[::1]", nil, true}, {"[::1]", true, "https://[::1]", nil, true}, {"[::1]:80", false, "http://[::1]:80", nil, true}, {"[::1]:443", true, "https://[::1]:443", nil, true}, {"[::1]:9000", false, "http://[::1]:9000", nil, true}, {"[::1]:9000", true, "https://[::1]:9000", nil, true}, {"13333.123123.-", true, "", errInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "13333.123123.-")), false}, {"13333.123123.-", true, "", errInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "13333.123123.-")), false}, {"s3.aamzza.-", true, "", errInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "s3.aamzza.-")), false}, {"", true, "", errInvalidArgument("Endpoint: does not follow ip address or domain name standards."), false}, } for i, testCase := range testCases { result, err := getEndpointURL(testCase.endPoint, testCase.secure) if err != nil && testCase.shouldPass { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) } if err == nil && !testCase.shouldPass { t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) } // Failed as expected, but does it fail for the expected reason. if err != nil && !testCase.shouldPass { if err.Error() != testCase.err.Error() { t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) } } // Test passes as expected, but the output values are verified for correctness here. if err == nil && testCase.shouldPass { if testCase.result != result.String() { t.Errorf("Test %d: Expected the result Url to be \"%s\", but found \"%s\" instead", i+1, testCase.result, result.String()) } } } } // Tests validate end point validator. func TestIsValidEndpointURL(t *testing.T) { testCases := []struct { url string err error // Flag indicating whether the test is expected to pass or not. shouldPass bool }{ {"", errInvalidArgument("Endpoint url cannot be empty."), false}, {"https://s3.amazonaws.com", nil, true}, {"https://s3.cn-north-1.amazonaws.com.cn", nil, true}, {"https://s3-us-gov-west-1.amazonaws.com", nil, true}, {"https://s3-fips-us-gov-west-1.amazonaws.com", nil, true}, {"https://s3-fips.us-gov-west-1.amazonaws.com", nil, true}, {"https://s3-fips.us-gov-east-1.amazonaws.com", nil, true}, {"https://s3.amazonaws.com/", nil, true}, {"https://storage.googleapis.com/", nil, true}, {"https://z3.amazonaws.com", nil, true}, {"https://mybalancer.us-east-1.elb.amazonaws.com", nil, true}, {"192.168.1.1", errInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, {"https://amazon.googleapis.com/", errInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'."), false}, {"https://storage.googleapis.com/bucket/", errInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, {"https://s3.amazonaws.com/bucket/object", errInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, {"https://.s3server.example.com/", errInvalidArgument("Endpoint: .s3server.example.com does not follow ip address or domain name standards."), false}, {"https://s3server.example_/", errInvalidArgument("Endpoint: s3server.example_ does not follow ip address or domain name standards."), false}, {"https://_s3server.example.com/", errInvalidArgument("Endpoint: _s3server.example.com does not follow ip address or domain name standards."), false}, } for i, testCase := range testCases { var u url.URL if testCase.url == "" { u = sentinelURL } else { u1, err := url.Parse(testCase.url) if err != nil { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err) } u = *u1 } err := isValidEndpointURL(u) if err != nil && testCase.shouldPass { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err) } if err == nil && !testCase.shouldPass { t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err) } // Failed as expected, but does it fail for the expected reason. if err != nil && !testCase.shouldPass { if err.Error() != testCase.err.Error() { t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err, err) } } } } func TestDefaultBucketLocation(t *testing.T) { testCases := []struct { endpointURL url.URL regionOverride string expectedLocation string }{ // Region override is set URL is ignored. - Test 1. { endpointURL: url.URL{Host: "s3-fips-us-gov-west-1.amazonaws.com"}, regionOverride: "us-west-1", expectedLocation: "us-west-1", }, // No region override, url based preferenced is honored - Test 2. { endpointURL: url.URL{Host: "s3-fips-us-gov-west-1.amazonaws.com"}, regionOverride: "", expectedLocation: "us-gov-west-1", }, // Region override is honored - Test 3. { endpointURL: url.URL{Host: "s3.amazonaws.com"}, regionOverride: "us-west-1", expectedLocation: "us-west-1", }, // China region should be honored, region override not provided. - Test 4. { endpointURL: url.URL{Host: "s3.cn-north-1.amazonaws.com.cn"}, regionOverride: "", expectedLocation: "cn-north-1", }, // China region should be honored, region override not provided. - Test 5. { endpointURL: url.URL{Host: "s3.cn-northwest-1.amazonaws.com.cn"}, regionOverride: "", expectedLocation: "cn-northwest-1", }, // No region provided, no standard region strings provided as well. - Test 6. { endpointURL: url.URL{Host: "s3.amazonaws.com"}, regionOverride: "", expectedLocation: "us-east-1", }, } for i, testCase := range testCases { retLocation := getDefaultLocation(testCase.endpointURL, testCase.regionOverride) if testCase.expectedLocation != retLocation { t.Errorf("Test %d: Expected location %s, got %s", i+1, testCase.expectedLocation, retLocation) } } } // Tests validate the expiry time validator. func TestIsValidExpiry(t *testing.T) { testCases := []struct { // Input. duration time.Duration // Expected result. err error // Flag to indicate whether the test should pass. shouldPass bool }{ {100 * time.Millisecond, errInvalidArgument("Expires cannot be lesser than 1 second."), false}, {604801 * time.Second, errInvalidArgument("Expires cannot be greater than 7 days."), false}, {0 * time.Second, errInvalidArgument("Expires cannot be lesser than 1 second."), false}, {1 * time.Second, nil, true}, {10000 * time.Second, nil, true}, {999 * time.Second, nil, true}, } for i, testCase := range testCases { err := isValidExpiry(testCase.duration) if err != nil && testCase.shouldPass { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) } if err == nil && !testCase.shouldPass { t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) } // Failed as expected, but does it fail for the expected reason. if err != nil && !testCase.shouldPass { if err.Error() != testCase.err.Error() { t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) } } } } // Tests validate the bucket name validator. func TestIsValidBucketName(t *testing.T) { testCases := []struct { // Input. bucketName string // Expected result. err error // Flag to indicate whether test should Pass. shouldPass bool }{ {".mybucket", errors.New("Bucket name contains invalid characters"), false}, {"mybucket.", errors.New("Bucket name contains invalid characters"), false}, {"mybucket-", errors.New("Bucket name contains invalid characters"), false}, {"my", errors.New("Bucket name cannot be shorter than 3 characters"), false}, {"", errors.New("Bucket name cannot be empty"), false}, {"my..bucket", errors.New("Bucket name contains invalid characters"), false}, {"my.bucket.com", nil, true}, {"my-bucket", nil, true}, {"123my-bucket", nil, true}, } for i, testCase := range testCases { err := s3utils.CheckValidBucketName(testCase.bucketName) if err != nil && testCase.shouldPass { t.Errorf("Test %d: Expected to pass, but failed with: %s", i+1, err.Error()) } if err == nil && !testCase.shouldPass { t.Errorf("Test %d: Expected to fail with \"%s\", but passed instead", i+1, testCase.err.Error()) } // Failed as expected, but does it fail for the expected reason. if err != nil && !testCase.shouldPass { if err.Error() != testCase.err.Error() { t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Error()) } } } } // Tests if header is standard supported header func TestIsStandardHeader(t *testing.T) { testCases := []struct { // Input. header string // Expected result. expectedValue bool }{ {"content-encoding", true}, {"content-type", true}, {"cache-control", true}, {"content-disposition", true}, {"content-language", true}, {"random-header", false}, } for i, testCase := range testCases { actual := isStandardHeader(testCase.header) if actual != testCase.expectedValue { t.Errorf("Test %d: Expected to pass, but failed", i+1) } } } // Tests if header is server encryption header func TestIsSSEHeader(t *testing.T) { testCases := []struct { // Input. header string // Expected result. expectedValue bool }{ {"x-amz-server-side-encryption", true}, {"x-amz-server-side-encryption-aws-kms-key-id", true}, {"x-amz-server-side-encryption-context", true}, {"x-amz-server-side-encryption-customer-algorithm", true}, {"x-amz-server-side-encryption-customer-key", true}, {"x-amz-server-side-encryption-customer-key-MD5", true}, {"random-header", false}, } for i, testCase := range testCases { actual := isSSEHeader(testCase.header) if actual != testCase.expectedValue { t.Errorf("Test %d: Expected to pass, but failed", i+1) } } } // Tests if header is x-amz-meta or x-amz-acl func TestIsAmzHeader(t *testing.T) { testCases := []struct { // Input. header string // Expected result. expectedValue bool }{ {"x-amz-iv", false}, {"x-amz-key", false}, {"x-amz-matdesc", false}, {"x-amz-meta-x-amz-iv", true}, {"x-amz-meta-x-amz-key", true}, {"x-amz-meta-x-amz-matdesc", true}, {"x-amz-acl", true}, {"random-header", false}, } for i, testCase := range testCases { actual := isAmzHeader(testCase.header) if actual != testCase.expectedValue { t.Errorf("Test %d: Expected to pass, but failed", i+1) } } } // Tests if query parameter starts with "x-" and will be ignored by S3. func TestIsCustomQueryValue(t *testing.T) { testCases := []struct { // Input. queryParamKey string // Expected result. expectedValue bool }{ {"x-custom-key", true}, {"xcustom-key", false}, {"random-header", false}, } for i, testCase := range testCases { actual := isCustomQueryValue(testCase.queryParamKey) if actual != testCase.expectedValue { t.Errorf("Test %d: Expected to pass, but failed", i+1) } } } func TestFullObjectChecksum64(t *testing.T) { tests := []ChecksumType{ ChecksumCRC32, ChecksumCRC32C, ChecksumCRC64NVME, } for _, cs := range tests { t.Run(cs.String(), func(t *testing.T) { b := make([]byte, 1024000) rng := rand.New(rand.NewSource(time.Now().UnixNano())) rng.Read(b) sum := cs.EncodeToString want := sum(b) var parts []ObjectPart for len(b) > 0 { sz := rng.Intn(len(b) / 2) if len(b)-sz < 1024 { sz = len(b) } switch cs { case ChecksumCRC32: parts = append(parts, ObjectPart{PartNumber: len(parts) + 1, ChecksumCRC32: cs.EncodeToString(b[:sz]), Size: int64(sz)}) case ChecksumCRC32C: parts = append(parts, ObjectPart{PartNumber: len(parts) + 1, ChecksumCRC32C: cs.EncodeToString(b[:sz]), Size: int64(sz)}) case ChecksumCRC64NVME: parts = append(parts, ObjectPart{PartNumber: len(parts) + 1, ChecksumCRC64NVME: cs.EncodeToString(b[:sz]), Size: int64(sz)}) } b = b[sz:] } gotCRC, err := cs.FullObjectChecksum(parts) if err != nil { t.Fatal(err) } if gotCRC.Encoded() != want { t.Errorf("Checksum %v does not match the expected CRC got:%s want:%s", cs.String(), gotCRC.Encoded(), want) } }) } } func TestExtractObjMetadata(t *testing.T) { tests := []struct { name string header http.Header want http.Header }{ { name: "Test with valid header", header: http.Header{ "X-Minio-Meta-Test": []string{"test"}, }, want: http.Header{ "X-Minio-Meta-Test": []string{"test"}, }, }, { name: "Test with valid header with QEncoding characters", header: http.Header{ "X-Minio-Meta-Test": []string{mime.QEncoding.Encode("UTF-8", "öha, das")}, }, want: http.Header{ "X-Minio-Meta-Test": []string{"öha, das"}, }, }, { name: "Test with valid header with BEncoding characters", header: http.Header{ "X-Minio-Meta-Test": []string{mime.BEncoding.Encode("UTF-8", "öha, das")}, }, want: http.Header{ "X-Minio-Meta-Test": []string{"öha, das"}, }, }, { name: "Test with valid header with multi-QEncoding characters", header: http.Header{ "X-Minio-Meta-Test": []string{mime.QEncoding.Encode("UTF-8", strings.Repeat("öha, das", 100))}, }, want: http.Header{ "X-Minio-Meta-Test": []string{strings.Repeat("öha, das", 100)}, }, }, { name: "Test with valid header with multi-BEncoding characters", header: http.Header{ "X-Minio-Meta-Test": []string{mime.BEncoding.Encode("UTF-8", strings.Repeat("öha, das", 100))}, }, want: http.Header{ "X-Minio-Meta-Test": []string{strings.Repeat("öha, das", 100)}, }, }, { name: "Test with valid header with multi-BEncoding characters", header: http.Header{ "X-Minio-Meta-Test": []string{mime.BEncoding.Encode("UTF-8", strings.Repeat("öha, das", 100)), mime.BEncoding.Encode("UTF-8", strings.Repeat("öha, das123", 100))}, }, want: http.Header{ "X-Minio-Meta-Test": []string{strings.Repeat("öha, das", 100), strings.Repeat("öha, das123", 100)}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := extractObjMetadata(tt.header) if !reflect.DeepEqual(got, tt.want) { t.Errorf("extractObjMetadata() = %v, want %v", got, tt.want) } }) } }