Skip to content

Commit 8bbad38

Browse files
flangnerflangner
authored
Initial upload
Full-featured 1.0 S3 benchmark tool.
1 parent d47827c commit 8bbad38

File tree

8 files changed

+1115
-0
lines changed

8 files changed

+1115
-0
lines changed

‎README

+103
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
S3Bench
2+
3+
S3Bench is a light-weight S3 service benchmark tool, for measuring
4+
latencies and throughput for object up- and downloads. This software
5+
uses the AWS S3 Java SDK as foundation and provides basic CLI and
6+
properties file-based customization for benchmark workloads.
7+
8+
9+
Getting started
10+
11+
>./s3bench.sh s3.domain.com mybucket accessKeyId secret*** [s3bench.properties]
12+
13+
14+
Customization options (s3bench.properties)
15+
16+
The given values match the fallback default properties, if the optional
17+
properties file command line argument was not specified.
18+
19+
# Determines whether requests use secure or plain-text connections.
20+
# true: requests are using HTTPS (verifiable certificates only)
21+
# false: plain-text HTTP requests
22+
use_ssl=false
23+
24+
# Specifies how buckets are encoded in the client request URLs.
25+
# true: via URL path (i.e. http://s3.domain.com/mybucket)
26+
# false: via sub-domain (i.e. http://mybucket.s3.domain.com)
27+
path_style=true
28+
29+
# MD5 checksum verification by the client prolong the overall request latency.
30+
# true: verification is enabled
31+
# false: verification is disabled
32+
client_md5_verification=false
33+
34+
# If the bucket retrieved as a command line option is not existent this flag
35+
# determines, whether this tool may create it.
36+
# true: s3bench may create the target bucket
37+
# false: s3bench will fail if the target bucket does not exist
38+
create_bucket=true
39+
40+
# The number of objects created during a single stream benchmark.
41+
sample_count=10
42+
43+
# Object payload is transmitted unsigned, removing the need of server side
44+
# checksum generation (fast).
45+
benchmark_unsigned=true
46+
47+
# Object payload is signed (and verified at the server) via AWS S3 signature V4
48+
# chunked uploads.
49+
benchmark_chunked=true
50+
51+
# Object payload is SHA256 pre-signed and verified by the server.
52+
benchmark_presigned=true
53+
54+
# Any objects in the target bucket are removed after the run.
55+
# true: remove _any_ objects in the target bucket
56+
# false: leave the bucket as is after finishing the benchmark
57+
clean_after_run=true
58+
59+
# Uses large objects to obtain throughput key figures.
60+
# true: run throughput workloads
61+
# false: skip throughput workloads
62+
benchmark_tp=true
63+
64+
# Object size in bytes during throughput benchmarks in bytes (limit: < 2^31).
65+
tp_object_size=1073741824
66+
67+
# Uses small objects to obtain latency key figures.
68+
# true: run latency workloads
69+
# false: skip latency workloads
70+
benchmark_ops=true
71+
72+
# Object size in bytes during latency benchmarks in bytes (limit: < 2^31).
73+
ops_object_size=4096
74+
75+
# Run single-stream workloads for sequential performance benchmarks.
76+
# true: run sequential workloads
77+
# false: skip sequential workloads
78+
benchmark_single_stream=true
79+
80+
# Run multi-stream workloads for parallel performance benchmarks.
81+
# true: run parallel workloads
82+
# false: skip parallel workloads
83+
benchmark_multi_stream=true
84+
85+
# The runtime of a single stream benchmark in seconds.
86+
multi_stream_runtime_s=600
87+
88+
# Number of clients during a multi-stream throughput benchmark.
89+
multi_stream_tp_clients=2
90+
91+
# Number of threads per client during a multi-stream throughput benchmark.
92+
multi_stream_tp_threads=10
93+
94+
# Number of clients during multi-stream latency benchmarks.
95+
multi_stream_ops_clients=10
96+
97+
# Number of threads per client during multi-stream latency benchmarks.
98+
multi_stream_ops_threads=10
99+
100+
# Run download object benchmarks after object upload workloads.
101+
# true: run download object workloads
102+
# false: skip download object workloads
103+
benchmark_get=true

‎pom.xml

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
5+
<groupId>com.quobyte</groupId>
6+
<artifactId>s3bench</artifactId>
7+
<version>1.0.0</version>
8+
9+
<name>s3bench</name>
10+
<url>http://maven.apache.org</url>
11+
12+
<properties>
13+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
14+
<maven.compiler.source>1.8</maven.compiler.source>
15+
<maven.compiler.target>1.8</maven.compiler.target>
16+
</properties>
17+
18+
<dependencies>
19+
<dependency>
20+
<groupId>com.amazonaws</groupId>
21+
<artifactId>aws-java-sdk-s3</artifactId>
22+
</dependency>
23+
</dependencies>
24+
<dependencyManagement>
25+
<dependencies>
26+
<dependency>
27+
<groupId>com.amazonaws</groupId>
28+
<artifactId>aws-java-sdk-s3</artifactId>
29+
<version>1.11.186</version>
30+
</dependency>
31+
</dependencies>
32+
</dependencyManagement>
33+
<build>
34+
<plugins>
35+
<plugin>
36+
<groupId>org.apache.maven.plugins</groupId>
37+
<artifactId>maven-assembly-plugin</artifactId>
38+
<version>3.1.0</version>
39+
<executions>
40+
<execution>
41+
<id>dist</id>
42+
<phase>package</phase>
43+
<goals>
44+
<goal>single</goal>
45+
</goals>
46+
<configuration>
47+
<descriptors>
48+
<descriptor>src/main/assembly/dist.xml</descriptor>
49+
</descriptors>
50+
</configuration>
51+
</execution>
52+
</executions>
53+
</plugin>
54+
</plugins>
55+
</build>
56+
57+
<organization>
58+
<name>Quobyte Inc.</name>
59+
<url>https://www.quobyte.com</url>
60+
</organization>
61+
<description>S3Bench is a light-weight S3 service benchmark tool, for measuring latencies and throughput for object up- and downloads. This software uses the AWS S3 Java SDK as foundation and provides basic CLI and properties file-based customization for benchmark workloads.
62+
63+
</description>
64+
<issueManagement>
65+
<url>https://github.com/flangner/s3bench/issues</url>
66+
<system>GitHub</system>
67+
</issueManagement>
68+
</project>

‎src/main/assembly/dist.xml

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
2+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
4+
<id>bin</id>
5+
6+
<formats>
7+
<format>tar.gz</format>
8+
</formats>
9+
10+
<dependencySets>
11+
<dependencySet>
12+
<useProjectArtifact>false</useProjectArtifact>
13+
<outputDirectory>lib</outputDirectory>
14+
<unpack>false</unpack>
15+
</dependencySet>
16+
</dependencySets>
17+
18+
<fileSets>
19+
<fileSet>
20+
<directory>${project.build.scriptSourceDirectory}</directory>
21+
<outputDirectory>/</outputDirectory>
22+
<includes>
23+
<include>s3bench.*</include>
24+
</includes>
25+
</fileSet>
26+
<fileSet>
27+
<directory>${project.build.directory}</directory>
28+
<outputDirectory>bin</outputDirectory>
29+
<includes>
30+
<include>*.jar</include>
31+
</includes>
32+
</fileSet>
33+
<fileSet>
34+
<directory>${project.basedir}</directory>
35+
<outputDirectory>/</outputDirectory>
36+
<includes>
37+
<include>README*</include>
38+
<include>LICENSE*</include>
39+
</includes>
40+
</fileSet>
41+
</fileSets>
42+
43+
</assembly>

‎src/main/java/com/quobyte/s3bench/Configuration.java

+176
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
/*
2+
* Copyright 2017 Quobyte Inc. All rights reserved.
3+
* Author: flangner@quobyte.com
4+
*/
5+
package com.quobyte.s3bench;
6+
7+
import java.io.File;
8+
import java.io.FileInputStream;
9+
import java.io.IOException;
10+
import java.io.InputStream;
11+
import java.util.Properties;
12+
13+
class Configuration {
14+
15+
static enum Mode {
16+
CHUNKED("benchmark_chunked"),
17+
PRESIGNED("benchmark_presigned"),
18+
UNSIGNED("benchmark_unsigned");
19+
20+
private final String propertiesKey;
21+
22+
private Mode(String propertiesKey) {
23+
this.propertiesKey = propertiesKey;
24+
}
25+
}
26+
27+
private static final String DEFAULT_URI = "/com/quobyte/s3bench/default.properties";
28+
29+
/**
30+
* Fallback, if default.properties could not be loaded.
31+
*/
32+
private static final boolean DEFAULT_CLIENT_MD5_VERIFICATION = false;
33+
private static final int DEFAULT_SAMPLE_COUNT = 10;
34+
private static final int DEFAULT_OPS_OBJECT_SIZE = 4 * 1024;
35+
private static final int DEFAULT_TP_OBJECT_SIZE = 128 * 1024 * 1024;
36+
private static final int DEFAULT_MULTI_OPS_CLIENTS = 10;
37+
private static final int DEFAULT_MULTI_OPS_THREADS = 10;
38+
private static final int DEFAULT_MULTI_TP_CLIENTS = 2;
39+
private static final int DEFAULT_MULTI_TP_THREADS = 5;
40+
private static final int DEFAULT_MULTI_STREAM_RUNTIME_S = 60;
41+
42+
private static final boolean DEFAULT_BENCHMARK_UNSIGNED = true;
43+
private static final boolean DEFAULT_BENCHMARK_PRESIGNED = true;
44+
private static final boolean DEFAULT_BENCHMARK_CHUNKED = true;
45+
private static final boolean DEFAULT_BENCHMARK_TP = true;
46+
private static final boolean DEFAULT_BENCHMARK_OPS = true;
47+
private static final boolean DEFAULT_BENCHMARK_GET = true;
48+
private static final boolean DEFAULT_BENCHMARK_SINGLE_STREAM = true;
49+
private static final boolean DEFAULT_BENCHMARK_MULTI_STREAM = true;
50+
private static final boolean DEFAULT_PATH_STYLE = true;
51+
private static final boolean DEFAULT_USE_SSL = false;
52+
53+
private static final boolean DEFAULT_CREATE_BUCKET = true;
54+
private static final boolean DEFAULT_CLEAN_AFTER_RUN = false;
55+
56+
private final Properties properties;
57+
58+
Configuration(String path) {
59+
properties = new Properties();
60+
if (path != null && !path.isEmpty()) {
61+
try (FileInputStream fis = new FileInputStream(new File(path))) {
62+
properties.load(fis);
63+
} catch (IOException e) {
64+
System.err.println("Could not load properties from " + path + " because " + e);
65+
}
66+
} else {
67+
try (InputStream fis = getClass().getResourceAsStream(DEFAULT_URI)) {
68+
properties.load(fis);
69+
} catch (IOException e) {
70+
System.err.println("Could not load default properties from " + DEFAULT_URI + " because "
71+
+ e);
72+
}
73+
}
74+
}
75+
76+
boolean isClientMd5VerificationEnabled() {
77+
return getBooleanProperty("client_md5_verification", DEFAULT_CLIENT_MD5_VERIFICATION);
78+
}
79+
80+
int getSampleCount() {
81+
return getIntProperty("sample_count", DEFAULT_SAMPLE_COUNT);
82+
}
83+
84+
boolean benchmarkUnsigned() {
85+
return getBooleanProperty(Mode.UNSIGNED.propertiesKey, DEFAULT_BENCHMARK_UNSIGNED);
86+
}
87+
88+
boolean benchmarkPresigned() {
89+
return getBooleanProperty(Mode.PRESIGNED.propertiesKey, DEFAULT_BENCHMARK_PRESIGNED);
90+
}
91+
92+
boolean benchmarkChunked() {
93+
return getBooleanProperty(Mode.CHUNKED.propertiesKey, DEFAULT_BENCHMARK_CHUNKED);
94+
}
95+
96+
boolean benchmarkTp() {
97+
return getBooleanProperty("benchmark_tp", DEFAULT_BENCHMARK_TP);
98+
}
99+
100+
boolean benchmarkOps() {
101+
return getBooleanProperty("benchmark_ops", DEFAULT_BENCHMARK_OPS);
102+
}
103+
104+
boolean benchmarkSingleStream() {
105+
return getBooleanProperty("benchmark_single_stream", DEFAULT_BENCHMARK_SINGLE_STREAM);
106+
}
107+
108+
boolean benchmarkMultiStream() {
109+
return getBooleanProperty("benchmark_multi_stream", DEFAULT_BENCHMARK_MULTI_STREAM);
110+
}
111+
112+
boolean benchmarkGet() {
113+
return getBooleanProperty("benchmark_get", DEFAULT_BENCHMARK_GET);
114+
}
115+
116+
boolean pathStyle() {
117+
return getBooleanProperty("path_style", DEFAULT_PATH_STYLE);
118+
}
119+
120+
boolean useSsl() {
121+
return getBooleanProperty("use_ssl", DEFAULT_USE_SSL);
122+
}
123+
124+
boolean createBucket() {
125+
return getBooleanProperty("create_bucket", DEFAULT_CREATE_BUCKET);
126+
}
127+
128+
129+
boolean cleanAfterRun() {
130+
return getBooleanProperty("clean_after_run", DEFAULT_CLEAN_AFTER_RUN);
131+
}
132+
133+
int getOpsObjectSize() {
134+
return getIntProperty("ops_object_size", DEFAULT_OPS_OBJECT_SIZE);
135+
}
136+
137+
int getTpObjectSize() {
138+
return getIntProperty("tp_object_size", DEFAULT_TP_OBJECT_SIZE);
139+
}
140+
141+
int getMultiOpsClients() {
142+
return getIntProperty("multi_stream_ops_clients", DEFAULT_MULTI_OPS_CLIENTS);
143+
}
144+
145+
int getMultiOpsThreads() {
146+
return getIntProperty("multi_stream_ops_threads", DEFAULT_MULTI_OPS_THREADS);
147+
}
148+
149+
int getMultiTpClients() {
150+
return getIntProperty("multi_stream_tp_clients", DEFAULT_MULTI_TP_CLIENTS);
151+
}
152+
153+
int getMultiTpThreads() {
154+
return getIntProperty("multi_stream_tp_threads", DEFAULT_MULTI_TP_THREADS);
155+
}
156+
157+
int getMultiStreamRuntimeS() {
158+
return getIntProperty("multi_stream_runtime_s", DEFAULT_MULTI_STREAM_RUNTIME_S);
159+
}
160+
161+
private boolean getBooleanProperty(String key, boolean defaultValue) {
162+
try {
163+
return Boolean.valueOf((String) properties.get(key));
164+
} catch (Throwable t) {
165+
return defaultValue;
166+
}
167+
}
168+
169+
private int getIntProperty(String key, int defaultValue) {
170+
try {
171+
return Integer.valueOf((String) properties.get(key));
172+
} catch (Throwable t) {
173+
return defaultValue;
174+
}
175+
}
176+
}

0 commit comments

Comments
 (0)