GetFile returns reader and headers. Use source file when copying file

itchio · Nov 23, 2023 · b89b535 · b89b535
1 parent 0438203
commit b89b535
Show file tree

Hide file tree

Showing 7 changed files with 25 additions and 29 deletions.
diff --git a/zipserver/archive.go b/zipserver/archive.go
@@ -67,7 +67,7 @@ func (a *Archiver) fetchZip(ctx context.Context, key string) (string, error) {
 	fname := fetchZipFilename(a.Bucket, key)
 	fname = path.Join(tmpDir, fname)
 
-	src, err := a.Storage.GetFile(ctx, a.Bucket, key)
+	src, _, err := a.Storage.GetFile(ctx, a.Bucket, key)
 	if err != nil {
 		return "", errors.Wrap(err, 0)
 	}

diff --git a/zipserver/copy_handler.go b/zipserver/copy_handler.go
@@ -98,7 +98,7 @@ func copyHandler(w http.ResponseWriter, r *http.Request) error {
 
 		startTime := time.Now()
 
-		reader, err := storage.GetFile(jobCtx, config.Bucket, key)
+		reader, headers, err := storage.GetFile(jobCtx, config.Bucket, key)
 		defer reader.Close()
 
 		if err != nil {
@@ -112,7 +112,11 @@ func copyHandler(w http.ResponseWriter, r *http.Request) error {
 		// transfer the reader to s3
 		// TODO: get the actual mime type from the GetFile request
 		log.Print("Starting transfer: ", key)
-		checksumMd5, err := targetStorage.PutFile(jobCtx, config.S3Bucket, key, mReader, "application/octet-stream")
+		contentType := headers.Get("Content-Type")
+		if contentType == "" {
+			contentType = "application/octet-stream"
+		}
+		checksumMd5, err := targetStorage.PutFile(jobCtx, config.S3Bucket, key, mReader, contentType)
 
 		if err != nil {
 			log.Print("Failed to copy file: ", err)

diff --git a/zipserver/gcs_storage.go b/zipserver/gcs_storage.go
@@ -64,28 +64,28 @@ func (c *GcsStorage) url(bucket, key, logName string) string {
 }
 
 // GetFile returns a reader for the contents of resource at bucket/key
-func (c *GcsStorage) GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, error) {
+func (c *GcsStorage) GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, http.Header, error) {
 	httpClient, err := c.httpClient()
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	url := c.url(bucket, key, "GET")
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	res, err := httpClient.Do(req)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if res.StatusCode != 200 {
-		return nil, errors.New(res.Status + " " + url)
+		return nil, res.Header, errors.New(res.Status + " " + url)
 	}
 
-	return res.Body, nil
+	return res.Body, res.Header, nil
 }
 
 // PutFile uploads a file to GCS simply

diff --git a/zipserver/list_handler.go b/zipserver/list_handler.go
@@ -39,7 +39,7 @@ func listFromBucket(ctx context.Context, key string, w http.ResponseWriter, r *h
 		return err
 	}
 
-	reader, err := storage.GetFile(ctx, config.Bucket, key)
+	reader, _, err := storage.GetFile(ctx, config.Bucket, key)
 	if err != nil {
 		return err
 	}

diff --git a/zipserver/mem_storage.go b/zipserver/mem_storage.go
@@ -45,19 +45,18 @@ func (fs *MemStorage) objectPath(bucket, key string) string {
 	return fmt.Sprintf("%s/%s", bucket, key)
 }
 
-// GetFile implements Storage.GetFile for FsStorage
-func (fs *MemStorage) GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, error) {
+func (fs *MemStorage) GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, http.Header, error) {
 	fs.mutex.Lock()
 	defer fs.mutex.Unlock()
 
 	objectPath := fs.objectPath(bucket, key)
 
 	if obj, ok := fs.objects[objectPath]; ok {
-		return io.NopCloser(bytes.NewReader(obj.data)), nil
+		return io.NopCloser(bytes.NewReader(obj.data)), obj.headers, nil
 	}
 
 	err := fmt.Errorf("%s: object not found", objectPath)
-	return nil, errors.Wrap(err, 0)
+	return nil, nil, errors.Wrap(err, 0)
 }
 
 func (fs *MemStorage) getHeaders(bucket, key string) (http.Header, error) {
@@ -74,15 +73,13 @@ func (fs *MemStorage) getHeaders(bucket, key string) (http.Header, error) {
 	return nil, errors.Wrap(err, 0)
 }
 
-// PutFile implements Storage.PutFile for FsStorage
 func (fs *MemStorage) PutFile(ctx context.Context, bucket, key string, contents io.Reader, mimeType string) error {
 	return fs.PutFileWithSetup(ctx, bucket, key, contents, func(req *http.Request) error {
 		req.Header.Set("Content-Type", mimeType)
 		return nil
 	})
 }
 
-// PutFileWithSetup implements Storage.PutFileWithSetup for FsStorage
 func (fs *MemStorage) PutFileWithSetup(ctx context.Context, bucket, key string, contents io.Reader, setup StorageSetupFunc) error {
 	fs.mutex.Lock()
 	defer fs.mutex.Unlock()
@@ -117,7 +114,6 @@ func (fs *MemStorage) PutFileWithSetup(ctx context.Context, bucket, key string,
 	return nil
 }
 
-// DeleteFile implements Storage.DeleteFile for FsStorage
 func (fs *MemStorage) DeleteFile(ctx context.Context, bucket, key string) error {
 	fs.mutex.Lock()
 	defer fs.mutex.Unlock()

diff --git a/zipserver/serve_zip.go b/zipserver/serve_zip.go
@@ -45,7 +45,7 @@ func (mhh *memoryHttpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
 	ctx, cancel := context.WithTimeout(r.Context(), mhh.fileGetTimeout)
 	defer cancel()
 
-	reader, err := mhh.storage.GetFile(ctx, mhh.bucket, objectPath)
+	reader, headers, err := mhh.storage.GetFile(ctx, mhh.bucket, objectPath)
 	if err != nil {
 		printError(err)
 		w.WriteHeader(404)
@@ -54,17 +54,13 @@ func (mhh *memoryHttpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
 	}
 	defer reader.Close()
 
-	headers, err := mhh.storage.getHeaders(mhh.bucket, objectPath)
-	if err != nil {
-		dumpError(w, err)
-		return
-	}
-
-	log.Printf("Headers: %v", headers)
+	if headers != nil {
+		log.Printf("Headers: %v", headers)
 
-	for k, vv := range headers {
-		for _, v := range vv {
-			w.Header().Add(k, v)
+		for k, vv := range headers {
+			for _, v := range vv {
+				w.Header().Add(k, v)
+			}
 		}
 	}
 

diff --git a/zipserver/storage.go b/zipserver/storage.go
@@ -11,7 +11,7 @@ type StorageSetupFunc func(*http.Request) error
 
 // Storage is a place we can get files from, put files into, or delete files from
 type Storage interface {
-	GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, error)
+	GetFile(ctx context.Context, bucket, key string) (io.ReadCloser, http.Header, error)
 	PutFile(ctx context.Context, bucket, key string, contents io.Reader, mimeType string) error
 	PutFileWithSetup(ctx context.Context, bucket, key string, contents io.Reader, setup StorageSetupFunc) error
 	DeleteFile(ctx context.Context, bucket, key string) error