From b89b535267da81101475a08c60e422131f1dba50 Mon Sep 17 00:00:00 2001 From: leaf corcoran Date: Wed, 22 Nov 2023 18:41:55 -0800 Subject: [PATCH] GetFile returns reader and headers. Use source file when copying file --- zipserver/archive.go | 2 +- zipserver/copy_handler.go | 8 ++++++-- zipserver/gcs_storage.go | 12 ++++++------ zipserver/list_handler.go | 2 +- zipserver/mem_storage.go | 10 +++------- zipserver/serve_zip.go | 18 +++++++----------- zipserver/storage.go | 2 +- 7 files changed, 25 insertions(+), 29 deletions(-) diff --git a/zipserver/archive.go b/zipserver/archive.go index c2adb6c..7a97514 100644 --- a/zipserver/archive.go +++ b/zipserver/archive.go @@ -67,7 +67,7 @@ func (a *Archiver) fetchZip(ctx context.Context, key string) (string, error) { fname := fetchZipFilename(a.Bucket, key) fname = path.Join(tmpDir, fname) - src, err := a.Storage.GetFile(ctx, a.Bucket, key) + src, _, err := a.Storage.GetFile(ctx, a.Bucket, key) if err != nil { return "", errors.Wrap(err, 0) } diff --git a/zipserver/copy_handler.go b/zipserver/copy_handler.go index c81955c..dcfeb8e 100644 --- a/zipserver/copy_handler.go +++ b/zipserver/copy_handler.go @@ -98,7 +98,7 @@ func copyHandler(w http.ResponseWriter, r *http.Request) error { startTime := time.Now() - reader, err := storage.GetFile(jobCtx, config.Bucket, key) + reader, headers, err := storage.GetFile(jobCtx, config.Bucket, key) defer reader.Close() if err != nil { @@ -112,7 +112,11 @@ func copyHandler(w http.ResponseWriter, r *http.Request) error { // transfer the reader to s3 // TODO: get the actual mime type from the GetFile request log.Print("Starting transfer: ", key) - checksumMd5, err := targetStorage.PutFile(jobCtx, config.S3Bucket, key, mReader, "application/octet-stream") + contentType := headers.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + checksumMd5, err := targetStorage.PutFile(jobCtx, config.S3Bucket, key, mReader, contentType) if err != nil { log.Print("Failed to copy file: ", err) diff --git a/zipserver/gcs_storage.go b/zipserver/gcs_storage.go index 2f8e752..b7003c5 100644 --- a/zipserver/gcs_storage.go +++ b/zipserver/gcs_storage.go @@ -64,28 +64,28 @@ func (c *GcsStorage) url(bucket, key, logName string) string { } // GetFile returns a reader for the contents of resource at bucket/key -func (c *GcsStorage) GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, error) { +func (c *GcsStorage) GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, http.Header, error) { httpClient, err := c.httpClient() if err != nil { - return nil, err + return nil, nil, err } url := c.url(bucket, key, "GET") req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { - return nil, err + return nil, nil, err } res, err := httpClient.Do(req) if err != nil { - return nil, err + return nil, nil, err } if res.StatusCode != 200 { - return nil, errors.New(res.Status + " " + url) + return nil, res.Header, errors.New(res.Status + " " + url) } - return res.Body, nil + return res.Body, res.Header, nil } // PutFile uploads a file to GCS simply diff --git a/zipserver/list_handler.go b/zipserver/list_handler.go index 53b3d5b..ca6165b 100644 --- a/zipserver/list_handler.go +++ b/zipserver/list_handler.go @@ -39,7 +39,7 @@ func listFromBucket(ctx context.Context, key string, w http.ResponseWriter, r *h return err } - reader, err := storage.GetFile(ctx, config.Bucket, key) + reader, _, err := storage.GetFile(ctx, config.Bucket, key) if err != nil { return err } diff --git a/zipserver/mem_storage.go b/zipserver/mem_storage.go index f089e4a..2acc669 100644 --- a/zipserver/mem_storage.go +++ b/zipserver/mem_storage.go @@ -45,19 +45,18 @@ func (fs *MemStorage) objectPath(bucket, key string) string { return fmt.Sprintf("%s/%s", bucket, key) } -// GetFile implements Storage.GetFile for FsStorage -func (fs *MemStorage) GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, error) { +func (fs *MemStorage) GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, http.Header, error) { fs.mutex.Lock() defer fs.mutex.Unlock() objectPath := fs.objectPath(bucket, key) if obj, ok := fs.objects[objectPath]; ok { - return io.NopCloser(bytes.NewReader(obj.data)), nil + return io.NopCloser(bytes.NewReader(obj.data)), obj.headers, nil } err := fmt.Errorf("%s: object not found", objectPath) - return nil, errors.Wrap(err, 0) + return nil, nil, errors.Wrap(err, 0) } func (fs *MemStorage) getHeaders(bucket, key string) (http.Header, error) { @@ -74,7 +73,6 @@ func (fs *MemStorage) getHeaders(bucket, key string) (http.Header, error) { return nil, errors.Wrap(err, 0) } -// PutFile implements Storage.PutFile for FsStorage func (fs *MemStorage) PutFile(ctx context.Context, bucket, key string, contents io.Reader, mimeType string) error { return fs.PutFileWithSetup(ctx, bucket, key, contents, func(req *http.Request) error { req.Header.Set("Content-Type", mimeType) @@ -82,7 +80,6 @@ func (fs *MemStorage) PutFile(ctx context.Context, bucket, key string, contents }) } -// PutFileWithSetup implements Storage.PutFileWithSetup for FsStorage func (fs *MemStorage) PutFileWithSetup(ctx context.Context, bucket, key string, contents io.Reader, setup StorageSetupFunc) error { fs.mutex.Lock() defer fs.mutex.Unlock() @@ -117,7 +114,6 @@ func (fs *MemStorage) PutFileWithSetup(ctx context.Context, bucket, key string, return nil } -// DeleteFile implements Storage.DeleteFile for FsStorage func (fs *MemStorage) DeleteFile(ctx context.Context, bucket, key string) error { fs.mutex.Lock() defer fs.mutex.Unlock() diff --git a/zipserver/serve_zip.go b/zipserver/serve_zip.go index 2aa93a5..e444ea8 100644 --- a/zipserver/serve_zip.go +++ b/zipserver/serve_zip.go @@ -45,7 +45,7 @@ func (mhh *memoryHttpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) ctx, cancel := context.WithTimeout(r.Context(), mhh.fileGetTimeout) defer cancel() - reader, err := mhh.storage.GetFile(ctx, mhh.bucket, objectPath) + reader, headers, err := mhh.storage.GetFile(ctx, mhh.bucket, objectPath) if err != nil { printError(err) w.WriteHeader(404) @@ -54,17 +54,13 @@ func (mhh *memoryHttpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) } defer reader.Close() - headers, err := mhh.storage.getHeaders(mhh.bucket, objectPath) - if err != nil { - dumpError(w, err) - return - } - - log.Printf("Headers: %v", headers) + if headers != nil { + log.Printf("Headers: %v", headers) - for k, vv := range headers { - for _, v := range vv { - w.Header().Add(k, v) + for k, vv := range headers { + for _, v := range vv { + w.Header().Add(k, v) + } } } diff --git a/zipserver/storage.go b/zipserver/storage.go index 6cd07b0..a75ff54 100644 --- a/zipserver/storage.go +++ b/zipserver/storage.go @@ -11,7 +11,7 @@ type StorageSetupFunc func(*http.Request) error // Storage is a place we can get files from, put files into, or delete files from type Storage interface { - GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, error) + GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, http.Header, error) PutFile(ctx context.Context, bucket, key string, contents io.Reader, mimeType string) error PutFileWithSetup(ctx context.Context, bucket, key string, contents io.Reader, setup StorageSetupFunc) error DeleteFile(ctx context.Context, bucket, key string) error