diff --git a/COSBench-User-Guide.odt b/COSBench-User-Guide.odt index ad37ec50..2457ec0c 100644 Binary files a/COSBench-User-Guide.odt and b/COSBench-User-Guide.odt differ diff --git a/README.md b/README.md index acaf4e44..f3eaef93 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ COSBench - Cloud Object Storage Benchmark ========================================= -COSBench is a benchmarking tool to measure the performance of Cloud Object Storage services. Object storage is an +COSBench is a benchmarking tool to measure the performance of Cloud Object Storage services. Object storage is an emerging technology that is different from traditional file systems (e.g., NFS) or block device systems (e.g., iSCSI). Amazon S3 and Openstack* swift are well-known object storage solutions. -COSBench now supports OpenStack* Swift and Amplidata v2.3, 2.5 and 3.1, as well as custom adaptors. +COSBench now supports OpenStack* Swift, Amazon* S3, Amplidata v2.3, 2.5 and 3.1, Scality*, Ceph, CDMI, Google* Cloud Storage, Aliyun OSS as well as custom adaptors. Important Notice and Contact Information @@ -14,11 +14,9 @@ Important Notice and Contact Information a) COSBench is not a product, and it does not have a full-time support team. Before you use this tool, please understand the need to invest enough effort to learn how to use it effectively and to address possible bugs. -b) To help COSBench develop further, please become an active member of the community and consider giving back by making +b) To help COSBench develop further, please become an active member of the community and consider giving back by making contributions. -For other questions, contact jiangang.duan@intel.com or yaguang.wang@intel.com - Licensing --------- @@ -52,7 +50,6 @@ If a build from source code is needed, please refer to BUILD.md for details. Resources --------- -Blog: TBD Wiki: (https://github.com/intel-cloud/cosbench/wiki) @@ -70,5 +67,13 @@ COSBench-Workload-Generator: (https://github.com/giteshnandre/COSBench-Workload- COSBench-Plot: (https://github.com/icclab/cosbench-plot) +COSBench-Appliance: (https://susestudio.com/a/8Kp374/cosbench) + +COSBench Ansible Playbook: + +- (http://www.ksingh.co.in/blog/2016/05/29/deploy-cosbench-using-ansible/) +- (https://github.com/ksingh7/ansible-role-cosbench) +- (https://galaxy.ansible.com/ksingh7/cosbench/) + = END = diff --git a/VERSION b/VERSION index 44bb5d1f..2b7c5ae0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.1 \ No newline at end of file +0.4.2 diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/XferCountingInputStream.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/XferCountingInputStream.java index 78bb419a..3dc16ee5 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/XferCountingInputStream.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/generator/XferCountingInputStream.java @@ -42,14 +42,14 @@ public int read(byte[] b, int off, int len) throws IOException { private void recordTime() { if (this.isFirstByte) { - this.xferStart = System.currentTimeMillis(); + this.xferStart = System.nanoTime(); this.isFirstByte = false; } - this.xferEnd = System.currentTimeMillis(); + this.xferEnd = System.nanoTime(); } public long getXferTime() { - long xferTime = this.xferEnd - this.xferStart; + long xferTime = (this.xferEnd - this.xferStart) / 1000000; return xferTime > 0 ? xferTime : 0L; } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Deleter.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Deleter.java index cb686e04..ea2eef61 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Deleter.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Deleter.java @@ -69,7 +69,7 @@ public static Sample doDelete(String conName, String objName, if (Thread.interrupted()) throw new AbortedException(); - long start = System.currentTimeMillis(); + long start = System.nanoTime(); try { session.getApi().deleteObject(conName, objName, config); @@ -87,11 +87,10 @@ public static Sample doDelete(String conName, String objName, op.getSampleType(), op.getName(), false); } - long end = System.currentTimeMillis(); + long end = System.nanoTime(); - Date now = new Date(end); - return new Sample(now, op.getId(), op.getOpType(), op.getSampleType(), - op.getName(), true, end - start, 0L, 0L); + return new Sample(new Date(), op.getId(), op.getOpType(), op.getSampleType(), + op.getName(), true, (end - start) / 1000000, 0L, 0L); } } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/FileWriter.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/FileWriter.java index dedc41d8..e851619c 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/FileWriter.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/FileWriter.java @@ -137,7 +137,7 @@ public Sample doWrite(InputStream in, long length, String conName, String objNam XferCountingInputStream cin = new XferCountingInputStream(in); - long start = System.currentTimeMillis(); + long start = System.nanoTime(); try { session.getApi().createObject(conName, objName, cin, length, config); @@ -153,10 +153,9 @@ public Sample doWrite(InputStream in, long length, String conName, String objNam IOUtils.closeQuietly(cin); } - long end = System.currentTimeMillis(); + long end = System.nanoTime(); - Date now = new Date(end); - return new Sample(now, getId(), getOpType(), getSampleType(), - getName(), true, end - start, cin.getXferTime(), cin.getByteCount()); + return new Sample(new Date(), getId(), getOpType(), getSampleType(), + getName(), true, (end - start) / 1000000, cin.getXferTime(), cin.getByteCount()); } } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Lister.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Lister.java index 6dc4fef6..ac42261a 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Lister.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Lister.java @@ -1,5 +1,5 @@ -/** - +/** + Copyright 2013 Intel Corporation, All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,7 +12,7 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. +limitations under the License. */ package com.intel.cosbench.driver.operator; @@ -37,7 +37,7 @@ public class Lister extends AbstractOperator { public static final String OP_TYPE = "list"; private ObjectPicker objPicker = new ObjectPicker(); - + private byte buffer[] = new byte[1024*1024]; public Lister() { @@ -75,14 +75,14 @@ private Sample doList(OutputStream out, String conName, String objName, InputStream in = null; CountingOutputStream cout = new CountingOutputStream(out); doLogWarn(session.getLogger(), "listerrr: "+ conName + "/" + objName);//### - long start = System.currentTimeMillis(); + long start = System.nanoTime(); long xferTime = 0L; try { - doLogDebug(session.getLogger(), "worker "+ session.getIndex() + " List target " + conName + "/" + objName); + doLogDebug(session.getLogger(), "worker "+ session.getIndex() + " List target " + conName + "/" + objName); in = session.getApi().getList(conName, objName, config); - long xferStart = System.currentTimeMillis(); + long xferStart = System.nanoTime(); copyLarge(in, cout); - xferTime = System.currentTimeMillis() - xferStart; + xferTime = (System.nanoTime() - xferStart) / 1000000; } catch (StorageInterruptedException sie) { doLogErr(session.getLogger(), sie.getMessage(), sie); throw new AbortedException(); @@ -95,20 +95,16 @@ private Sample doList(OutputStream out, String conName, String objName, IOUtils.closeQuietly(in); IOUtils.closeQuietly(cout); } - long end = System.currentTimeMillis(); + long end = System.nanoTime(); - Date now = new Date(end); - return new Sample(now, getId(), getOpType(), getSampleType(), - getName(), true, end - start, xferTime, cout.getByteCount()); + return new Sample(new Date(), getId(), getOpType(), getSampleType(), + getName(), true, (end - start) / 1000000, xferTime, cout.getByteCount()); } public OutputStream copyLarge(InputStream input, OutputStream output) throws IOException { - for(int n = 0; -1 != (n = input.read(buffer));) - { - output.write(buffer, 0, n); - } + IOUtils.copyLarge(input, output); return output; } diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Reader.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Reader.java index 22e51e50..e0bcf285 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Reader.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Reader.java @@ -1,5 +1,5 @@ -/** - +/** + Copyright 2013 Intel Corporation, All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,8 +12,8 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. -*/ +limitations under the License. +*/ package com.intel.cosbench.driver.operator; @@ -32,18 +32,18 @@ /** * This class represents primitive READ operation. - * + * * @author ywang19, qzheng7 - * + * */ class Reader extends AbstractOperator { public static final String OP_TYPE = "read"; - + private boolean hashCheck = false; private ObjectPicker objPicker = new ObjectPicker(); - + private byte buffer[] = new byte[1024*1024]; public Reader() { @@ -82,51 +82,48 @@ private Sample doRead(OutputStream out, String conName, String objName, InputStream in = null; CountingOutputStream cout = new CountingOutputStream(out); - long start = System.currentTimeMillis(); + long start = System.nanoTime(); long xferTime = 0L; - long xferTimeCheck = 0L; try { in = session.getApi().getObject(conName, objName, config); - long xferStart = System.currentTimeMillis(); - if (!hashCheck){ + long xferStart = System.nanoTime(); + if (!hashCheck) { copyLarge(in, cout); - long xferEnd = System.currentTimeMillis(); - xferTime = xferEnd - xferStart; - } else if (!validateChecksum(conName, objName, session, in, cout, xferTimeCheck)) + } else if (!validateChecksum(conName, objName, session, in, cout)) { return new Sample(new Date(), getId(), getOpType(), getSampleType(), getName(), false); + } + long xferEnd = System.nanoTime(); + xferTime = (xferEnd - xferStart) / 1000000; } catch (StorageInterruptedException sie) { doLogErr(session.getLogger(), sie.getMessage(), sie); throw new AbortedException(); } catch (Exception e) { isUnauthorizedException(e, session); errorStatisticsHandle(e, session, conName + "/" + objName); - + return new Sample(new Date(), getId(), getOpType(), getSampleType(), getName(), false); } finally { IOUtils.closeQuietly(in); IOUtils.closeQuietly(cout); } - long end = System.currentTimeMillis(); + long end = System.nanoTime(); - Date now = new Date(end); - return new Sample(now, getId(), getOpType(), getSampleType(), - getName(), true, end - start, hashCheck ? xferTimeCheck : xferTime, cout.getByteCount()); + return new Sample(new Date(), getId(), getOpType(), getSampleType(), + getName(), true, (end - start)/1000000, + xferTime, cout.getByteCount()); } public OutputStream copyLarge(InputStream input, OutputStream output) throws IOException { - for(int n = 0; -1 != (n = input.read(buffer));) - { - output.write(buffer, 0, n); - } + IOUtils.copyLarge(input, output); return output; } - + private static boolean validateChecksum(String conName, String objName, - Session session, InputStream in, OutputStream out, long xferTimeCheck) + Session session, InputStream in, OutputStream out) throws IOException { HashUtil util; try { @@ -138,8 +135,7 @@ private static boolean validateChecksum(String conName, String objName, String storedHash = new String(); String calculatedHash = new String(); - - long xferStart = System.currentTimeMillis(); + int br1 = in.read(buf1); if (br1 <= hashLen) { @@ -176,8 +172,7 @@ private static boolean validateChecksum(String conName, String objName, br1 = br2; } } - xferTimeCheck = System.currentTimeMillis() - xferStart; - + if (!calculatedHash.equals(storedHash)) { if (storedHash.startsWith(HashUtil.GUARD)) { String err = diff --git a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Writer.java b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Writer.java index aa80f2e9..bc70a231 100644 --- a/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Writer.java +++ b/dev/cosbench-driver/src/com/intel/cosbench/driver/operator/Writer.java @@ -91,7 +91,7 @@ public static Sample doWrite(InputStream in, long length, String conName, throw new AbortedException(); XferCountingInputStream cin = new XferCountingInputStream(in); - long start = System.currentTimeMillis(); + long start = System.nanoTime(); try { session.getApi() @@ -110,10 +110,10 @@ public static Sample doWrite(InputStream in, long length, String conName, IOUtils.closeQuietly(cin); } - long end = System.currentTimeMillis(); - Date now = new Date(end); - return new Sample(now, op.getId(), op.getOpType(), op.getSampleType(), - op.getName(), true, end - start, cin.getXferTime(), cin.getByteCount()); + long end = System.nanoTime(); + return new Sample(new Date(), op.getId(), op.getOpType(), op.getSampleType(), + op.getName(), true, (end - start) / 1000000, + cin.getXferTime(), cin.getByteCount()); } /* * public static Sample doWrite(byte[] data, String conName, String objName, diff --git a/dev/cosbench-ecs/META-INF/.ecsmeta/.dirmeta b/dev/cosbench-ecs/META-INF/.ecsmeta/.dirmeta new file mode 100644 index 00000000..170229a4 --- /dev/null +++ b/dev/cosbench-ecs/META-INF/.ecsmeta/.dirmeta @@ -0,0 +1 @@ +{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":0,"contentType":"application/x-directory","httpExpires":null,"modificationTime":"Jun 23, 2016 4:03:00 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2016-06-20T21:16:47Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-23T21:02:46Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2016-06-20T21:16:47Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rwxr-xr-x","indexed":false}},"acl":null,"checksum":null,"expirationDate":null} \ No newline at end of file diff --git a/dev/cosbench-ecs/META-INF/.ecsmeta/MANIFEST.MF b/dev/cosbench-ecs/META-INF/.ecsmeta/MANIFEST.MF new file mode 100644 index 00000000..6d1e583e --- /dev/null +++ b/dev/cosbench-ecs/META-INF/.ecsmeta/MANIFEST.MF @@ -0,0 +1 @@ +{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":777,"contentType":"application/octet-stream","httpExpires":null,"modificationTime":"Jun 23, 2016 4:03:49 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2016-06-20T21:16:47Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-20T21:16:47Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2016-06-20T21:16:47Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rw-r--r--","indexed":false}},"acl":null,"checksum":null,"expirationDate":null} \ No newline at end of file diff --git a/dev/cosbench-ecs/META-INF/MANIFEST.MF b/dev/cosbench-ecs/META-INF/MANIFEST.MF new file mode 100644 index 00000000..476b4b12 --- /dev/null +++ b/dev/cosbench-ecs/META-INF/MANIFEST.MF @@ -0,0 +1,28 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: cosbench-ecs +Bundle-SymbolicName: cosbench-ecs +Bundle-Version: 0.4.2.0 +Bundle-Vendor: emc +Import-Package: com.intel.cosbench.api.auth, + com.intel.cosbench.api.context, + com.intel.cosbench.api.storage, + com.intel.cosbench.client.http, + com.intel.cosbench.config, + com.intel.cosbench.log +Bundle-ClassPath: ., + commons-codec-1.4.jar, + commons-logging-1.1.1.jar, + httpclient-4.5.jar, + httpcore-4.4.1.jar, + javax.mail-api-1.4.6.jar, + jdom2-2.0.5.jar, + jersey-apache-client4-1.18.3.jar, + jersey-client-1.18.3.jar, + jersey-core-1.18.3.jar, + log4j-1.2.17.jar, + lzma-sdk-4j-9.22.0.jar, + object-client-2.1.0.jar, + slf4j-api-1.7.5.jar, + smart-client-2.0.6.jar, + vipr-object-transformations-2.0.3.jar diff --git a/dev/cosbench-ecs/META-INF/spring/.ecsmeta/.dirmeta b/dev/cosbench-ecs/META-INF/spring/.ecsmeta/.dirmeta new file mode 100644 index 00000000..b1e05e39 --- /dev/null +++ b/dev/cosbench-ecs/META-INF/spring/.ecsmeta/.dirmeta @@ -0,0 +1 @@ +{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":0,"contentType":"application/x-directory","httpExpires":null,"modificationTime":"Jun 23, 2016 4:03:49 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2015-03-11T16:55:08Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-23T21:02:46Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2015-03-11T16:55:08Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rwxr-xr-x","indexed":false}},"acl":null,"checksum":null,"expirationDate":null} \ No newline at end of file diff --git a/dev/cosbench-ecs/META-INF/spring/.ecsmeta/plugin-context.xml b/dev/cosbench-ecs/META-INF/spring/.ecsmeta/plugin-context.xml new file mode 100644 index 00000000..65f0caea --- /dev/null +++ b/dev/cosbench-ecs/META-INF/spring/.ecsmeta/plugin-context.xml @@ -0,0 +1 @@ +{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":774,"contentType":"application/octet-stream","httpExpires":null,"modificationTime":"Jun 23, 2016 4:06:25 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2015-03-11T16:55:08Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-16T16:58:52Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2015-03-11T16:55:08Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rw-r--r--","indexed":false}},"acl":null,"checksum":null,"expirationDate":null} \ No newline at end of file diff --git a/dev/cosbench-ecs/META-INF/spring/plugin-context.xml b/dev/cosbench-ecs/META-INF/spring/plugin-context.xml new file mode 100644 index 00000000..14bfc6d5 --- /dev/null +++ b/dev/cosbench-ecs/META-INF/spring/plugin-context.xml @@ -0,0 +1,17 @@ + + + + + + + + + + \ No newline at end of file diff --git a/dev/cosbench-ecs/build.properties b/dev/cosbench-ecs/build.properties new file mode 100644 index 00000000..f26385c1 --- /dev/null +++ b/dev/cosbench-ecs/build.properties @@ -0,0 +1,20 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + .,\ + commons-codec-1.4.jar,\ + commons-logging-1.1.1.jar,\ + httpclient-4.5.jar,\ + httpcore-4.4.1.jar,\ + javax.mail-api-1.4.6.jar,\ + jdom2-2.0.5.jar,\ + jersey-apache-client4-1.18.3.jar,\ + jersey-client-1.18.3.jar,\ + jersey-core-1.18.3.jar,\ + log4j-1.2.17.jar,\ + lzma-sdk-4j-9.22.0.jar,\ + object-client-2.1.0.jar,\ + slf4j-api-1.7.5.jar,\ + smart-client-2.0.6.jar,\ + vipr-object-transformations-2.0.3.jar +jars.compile.order = . diff --git a/dev/cosbench-ecs/commons-codec-1.4.jar b/dev/cosbench-ecs/commons-codec-1.4.jar new file mode 100644 index 00000000..458d432d Binary files /dev/null and b/dev/cosbench-ecs/commons-codec-1.4.jar differ diff --git a/dev/cosbench-s3/commons-logging-1.1.1.jar b/dev/cosbench-ecs/commons-logging-1.1.1.jar similarity index 84% rename from dev/cosbench-s3/commons-logging-1.1.1.jar rename to dev/cosbench-ecs/commons-logging-1.1.1.jar index 8758a96b..1deef144 100644 Binary files a/dev/cosbench-s3/commons-logging-1.1.1.jar and b/dev/cosbench-ecs/commons-logging-1.1.1.jar differ diff --git a/dev/cosbench-ecs/cosbench-ecs.iml b/dev/cosbench-ecs/cosbench-ecs.iml new file mode 100644 index 00000000..481000c8 --- /dev/null +++ b/dev/cosbench-ecs/cosbench-ecs.iml @@ -0,0 +1,257 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dev/cosbench-ecs/httpclient-4.5.jar b/dev/cosbench-ecs/httpclient-4.5.jar new file mode 100644 index 00000000..970c9891 Binary files /dev/null and b/dev/cosbench-ecs/httpclient-4.5.jar differ diff --git a/dev/cosbench-ecs/httpcore-4.4.1.jar b/dev/cosbench-ecs/httpcore-4.4.1.jar new file mode 100644 index 00000000..99715b6a Binary files /dev/null and b/dev/cosbench-ecs/httpcore-4.4.1.jar differ diff --git a/dev/cosbench-ecs/javax.mail-api-1.4.6.jar b/dev/cosbench-ecs/javax.mail-api-1.4.6.jar new file mode 100755 index 00000000..bbb38767 Binary files /dev/null and b/dev/cosbench-ecs/javax.mail-api-1.4.6.jar differ diff --git a/dev/cosbench-ecs/jdom2-2.0.5.jar b/dev/cosbench-ecs/jdom2-2.0.5.jar new file mode 100644 index 00000000..b6996c72 Binary files /dev/null and b/dev/cosbench-ecs/jdom2-2.0.5.jar differ diff --git a/dev/cosbench-ecs/jersey-apache-client4-1.18.3.jar b/dev/cosbench-ecs/jersey-apache-client4-1.18.3.jar new file mode 100644 index 00000000..266b68e4 Binary files /dev/null and b/dev/cosbench-ecs/jersey-apache-client4-1.18.3.jar differ diff --git a/dev/cosbench-ecs/jersey-client-1.18.3.jar b/dev/cosbench-ecs/jersey-client-1.18.3.jar new file mode 100644 index 00000000..04ca66ec Binary files /dev/null and b/dev/cosbench-ecs/jersey-client-1.18.3.jar differ diff --git a/dev/cosbench-ecs/jersey-core-1.18.3.jar b/dev/cosbench-ecs/jersey-core-1.18.3.jar new file mode 100644 index 00000000..c380e92d Binary files /dev/null and b/dev/cosbench-ecs/jersey-core-1.18.3.jar differ diff --git a/dev/cosbench-ecs/log4j-1.2.17.jar b/dev/cosbench-ecs/log4j-1.2.17.jar new file mode 100644 index 00000000..1d425cf7 Binary files /dev/null and b/dev/cosbench-ecs/log4j-1.2.17.jar differ diff --git a/dev/cosbench-ecs/lzma-sdk-4j-9.22.0.jar b/dev/cosbench-ecs/lzma-sdk-4j-9.22.0.jar new file mode 100644 index 00000000..b1839df1 Binary files /dev/null and b/dev/cosbench-ecs/lzma-sdk-4j-9.22.0.jar differ diff --git a/dev/cosbench-ecs/object-client-2.1.0.jar b/dev/cosbench-ecs/object-client-2.1.0.jar new file mode 100644 index 00000000..40f72782 Binary files /dev/null and b/dev/cosbench-ecs/object-client-2.1.0.jar differ diff --git a/dev/cosbench-ecs/slf4j-api-1.7.5.jar b/dev/cosbench-ecs/slf4j-api-1.7.5.jar new file mode 100644 index 00000000..8f004d39 Binary files /dev/null and b/dev/cosbench-ecs/slf4j-api-1.7.5.jar differ diff --git a/dev/cosbench-ecs/smart-client-2.0.6.jar b/dev/cosbench-ecs/smart-client-2.0.6.jar new file mode 100644 index 00000000..e08bbc02 Binary files /dev/null and b/dev/cosbench-ecs/smart-client-2.0.6.jar differ diff --git a/dev/cosbench-ecs/src/.ecsmeta/.dirmeta b/dev/cosbench-ecs/src/.ecsmeta/.dirmeta new file mode 100644 index 00000000..51f85ea1 --- /dev/null +++ b/dev/cosbench-ecs/src/.ecsmeta/.dirmeta @@ -0,0 +1 @@ +{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":0,"contentType":"application/x-directory","httpExpires":null,"modificationTime":"Jun 23, 2016 4:03:00 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2014-11-19T20:38:22Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-23T21:02:46Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2014-11-19T20:38:22Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rwxr-xr-x","indexed":false}},"acl":null,"checksum":null,"expirationDate":null} \ No newline at end of file diff --git a/dev/cosbench-ecs/src/com/.ecsmeta/.dirmeta b/dev/cosbench-ecs/src/com/.ecsmeta/.dirmeta new file mode 100644 index 00000000..40e820d0 --- /dev/null +++ b/dev/cosbench-ecs/src/com/.ecsmeta/.dirmeta @@ -0,0 +1 @@ +{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":0,"contentType":"application/x-directory","httpExpires":null,"modificationTime":"Jun 23, 2016 4:03:50 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2014-11-19T20:38:22Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-23T21:02:46Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2014-11-19T20:38:22Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rwxr-xr-x","indexed":false}},"acl":null,"checksum":null,"expirationDate":null} \ No newline at end of file diff --git a/dev/cosbench-ecs/src/com/emc/.ecsmeta/.dirmeta b/dev/cosbench-ecs/src/com/emc/.ecsmeta/.dirmeta new file mode 100644 index 00000000..c931195c --- /dev/null +++ b/dev/cosbench-ecs/src/com/emc/.ecsmeta/.dirmeta @@ -0,0 +1 @@ +{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":0,"contentType":"application/x-directory","httpExpires":null,"modificationTime":"Jun 23, 2016 4:06:27 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2015-03-06T18:29:54Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-23T21:02:47Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2015-03-06T18:29:54Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rwxr-xr-x","indexed":false}},"acl":null,"checksum":null,"expirationDate":null} \ No newline at end of file diff --git a/dev/cosbench-ecs/src/com/emc/vipr/.ecsmeta/.dirmeta b/dev/cosbench-ecs/src/com/emc/vipr/.ecsmeta/.dirmeta new file mode 100644 index 00000000..bea511d9 --- /dev/null +++ b/dev/cosbench-ecs/src/com/emc/vipr/.ecsmeta/.dirmeta @@ -0,0 +1 @@ +{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":0,"contentType":"application/x-directory","httpExpires":null,"modificationTime":"Jun 23, 2016 4:11:56 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2014-11-19T20:38:22Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-23T21:02:47Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2014-11-19T20:38:22Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rwxr-xr-x","indexed":false}},"acl":null,"checksum":null,"expirationDate":null} \ No newline at end of file diff --git a/dev/cosbench-ecs/src/com/emc/vipr/cosbench/.ecsmeta/.dirmeta b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/.ecsmeta/.dirmeta new file mode 100644 index 00000000..0e619466 --- /dev/null +++ b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/.ecsmeta/.dirmeta @@ -0,0 +1 @@ +{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":0,"contentType":"application/x-directory","httpExpires":null,"modificationTime":"Jun 23, 2016 4:18:41 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2015-03-11T16:53:12Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-23T21:02:47Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2015-03-11T16:53:12Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rwxr-xr-x","indexed":false}},"acl":null,"checksum":null,"expirationDate":null} \ No newline at end of file diff --git a/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/.ecsmeta/.dirmeta b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/.ecsmeta/.dirmeta new file mode 100644 index 00000000..65793fef --- /dev/null +++ b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/.ecsmeta/.dirmeta @@ -0,0 +1 @@ +{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":0,"contentType":"application/x-directory","httpExpires":null,"modificationTime":"Jun 23, 2016 4:23:56 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2016-06-20T15:52:33Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-23T21:02:48Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2016-06-20T15:52:33Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rwxr-xr-x","indexed":false}},"acl":null,"checksum":null,"expirationDate":null} \ No newline at end of file diff --git a/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/.ecsmeta/ECSConstants.java b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/.ecsmeta/ECSConstants.java new file mode 100644 index 00000000..1c74319f --- /dev/null +++ b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/.ecsmeta/ECSConstants.java @@ -0,0 +1 @@ +{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":2529,"contentType":"application/octet-stream","httpExpires":null,"modificationTime":"Jun 23, 2016 4:30:40 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2016-05-24T15:54:45Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-20T15:23:45Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2016-05-24T15:54:45Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rw-r--r--","indexed":false}},"acl":null,"checksum":null,"expirationDate":null} \ No newline at end of file diff --git a/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/.ecsmeta/ECSStorage.java b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/.ecsmeta/ECSStorage.java new file mode 100644 index 00000000..4699571b --- /dev/null +++ b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/.ecsmeta/ECSStorage.java @@ -0,0 +1 @@ +{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":10046,"contentType":"application/octet-stream","httpExpires":null,"modificationTime":"Jun 23, 2016 4:30:40 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2016-06-13T22:12:05Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-16T16:58:53Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2016-06-13T22:12:05Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rw-r--r--","indexed":false}},"acl":null,"checksum":null,"expirationDate":null} \ No newline at end of file diff --git a/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/.ecsmeta/ECSStorageFactory.java b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/.ecsmeta/ECSStorageFactory.java new file mode 100644 index 00000000..738bd2fd --- /dev/null +++ b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/.ecsmeta/ECSStorageFactory.java @@ -0,0 +1 @@ +{"instanceClass":"com.emc.ecs.sync.model.SyncMetadata","cacheControl":null,"contentDisposition":null,"contentEncoding":"identity","contentLength":1150,"contentType":"application/octet-stream","httpExpires":null,"modificationTime":"Jun 23, 2016 4:30:40 PM","userMetadata":{"x-emc-crtime":{"key":"x-emc-crtime","value":"2015-05-15T18:49:33Z","indexed":false},"x-emc-posix-owner-name":{"key":"x-emc-posix-owner-name","value":"petera5","indexed":false},"x-emc-posix-group-owner-name":{"key":"x-emc-posix-group-owner-name","value":"CORP\\Domain Users","indexed":false},"x-emc-atime":{"key":"x-emc-atime","value":"2016-06-16T16:58:53Z","indexed":false},"x-emc-mtime":{"key":"x-emc-mtime","value":"2015-05-15T18:49:33Z","indexed":false},"x-emc-posix-mode":{"key":"x-emc-posix-mode","value":"rw-r--r--","indexed":false}},"acl":null,"checksum":null,"expirationDate":null} \ No newline at end of file diff --git a/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/ECSConstants.java b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/ECSConstants.java new file mode 100644 index 00000000..c2ea2e8f --- /dev/null +++ b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/ECSConstants.java @@ -0,0 +1,66 @@ +/* + * Copyright 2014-2016 EMC Corporation. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0.txt + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.emc.vipr.cosbench.ECSStor; + +/** + * Constants for use with EMC ECS Storage API plugin + * for COSBench. + * + * The EMC ECS plugin requires at least access key and + * secret key to be user-defined. If unspecified, ECS + * endpoint will default to s3.amazonaws.com with a connection + * timeout of 30000ms; all other values are empty by default. + */ +public interface ECSConstants { + + // Connection Timeout Control + public static final String CONN_TIMEOUT_KEY = "timeout"; + public static final String CONN_TIMEOUT_DEFAULT = "30000"; + + // Read Timeout Control; default is same as connection timeout + public static final String READ_TIMEOUT_KEY = "read_timeout"; + + // ECS Endpoint Constants + public static final String ENDPOINT_KEY = "endpoint"; + public static final String ENDPOINT_DEFAULT = "http://s3.amazonaws.com"; + + // ECS Access Key + public static final String AUTH_USERNAME_KEY = "accesskey"; + public static final String AUTH_USERNAME_DEFAULT = ""; + + // ECS Secret Key + public static final String AUTH_PASSWORD_KEY = "secretkey"; + public static final String AUTH_PASSWORD_DEFAULT = ""; + + // Proxy Port/Host Keys - currently unsupported +// public static final String PROXY_HOST_KEY = "proxyhost"; +// public static final String PROXY_PORT_KEY = "proxyport"; + + // Namespace Key + public static final String NAMESPACE_KEY = "namespace"; + + // Path Style Access Constants + public static final String PATH_STYLE_ACCESS_KEY = "path_style_access"; + public static final boolean PATH_STYLE_ACCESS_DEFAULT = false; + + // HTTP Client to use: apache/java + public static final String HTTP_CLIENT_KEY = "http_client"; + public static final String HTTP_CLIENT_DEFAULT = "apache"; + + // Whether to use the smart client (incl. client-side load balancing) + public static final String SMART_CLIENT_KEY = "use_smart_client"; + public static final boolean SMART_CLIENT_DEFAULT = true; +} \ No newline at end of file diff --git a/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/ECSStorage.java b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/ECSStorage.java new file mode 100644 index 00000000..18b03610 --- /dev/null +++ b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/ECSStorage.java @@ -0,0 +1,274 @@ +/* + * Copyright 2014-2016 EMC Corporation. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0.txt + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.emc.vipr.cosbench.ECSStor; + +import com.emc.object.Protocol; +import com.emc.object.Range; +import com.emc.object.s3.S3Client; +import com.emc.object.s3.S3Config; +import com.emc.object.s3.S3ObjectMetadata; +import com.emc.object.s3.jersey.S3JerseyClient; +import com.emc.object.s3.request.PutObjectRequest; +import com.intel.cosbench.api.context.AuthContext; +import com.intel.cosbench.api.storage.NoneStorage; +import com.intel.cosbench.config.Config; +import com.intel.cosbench.log.Logger; +import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; + +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; + +/** + * COSBench Storage API for EMC ECS Object Storage interface. + * Enables use of ECS smart client for load balancing. + */ +public class ECSStorage extends NoneStorage + implements ECSConstants { + + //Local environment variables + private String connTimeout; + private String readTimeout; + private String accessKey; + private String secretKey; + private String endpoint; + private String httpClient; + private boolean smartClient; + protected S3Client s3Client; + S3Config s3CliConfig; + + /** + * Empty constructor; does nothing. + */ + public ECSStorage() { + } + + /** + * Initialize this storage API using the provided congifuration + * and logger. + * + * @param config configuration to be used for this operation + * @param logger logger instance to for log information output + */ + public void init(Config config, Logger logger) { + + super.init(config, logger); + + //Initialize environment variables from config + connTimeout = config.get(CONN_TIMEOUT_KEY, CONN_TIMEOUT_DEFAULT); + if (config.get(READ_TIMEOUT_KEY, "") != "") { + readTimeout = config.get(READ_TIMEOUT_KEY, ""); + } + else readTimeout = connTimeout; + parms.put(CONN_TIMEOUT_KEY, connTimeout); + parms.put(READ_TIMEOUT_KEY, readTimeout); + + endpoint = config.get(ENDPOINT_KEY, ENDPOINT_DEFAULT); + accessKey = config.get(AUTH_USERNAME_KEY, AUTH_USERNAME_DEFAULT); + secretKey = config.get(AUTH_PASSWORD_KEY, AUTH_PASSWORD_DEFAULT); + boolean pathStyleAccess = config.getBoolean(PATH_STYLE_ACCESS_KEY, PATH_STYLE_ACCESS_DEFAULT); +// String proxyHost = config.get(PROXY_HOST_KEY, ""); +// String proxyPort = config.get(PROXY_PORT_KEY, ""); + String namespace = config.get(NAMESPACE_KEY, ""); + httpClient = config.get(HTTP_CLIENT_KEY, HTTP_CLIENT_DEFAULT); + smartClient = config.getBoolean(SMART_CLIENT_KEY, SMART_CLIENT_DEFAULT); + + //Put new info. to environment variables where applicable + parms.put(ENDPOINT_KEY, endpoint); + parms.put(AUTH_USERNAME_KEY, accessKey); + parms.put(AUTH_PASSWORD_KEY, secretKey); + parms.put(PATH_STYLE_ACCESS_KEY, pathStyleAccess); +// parms.put(PROXY_HOST_KEY, proxyHost); +// parms.put(PROXY_PORT_KEY, proxyPort); + parms.put(NAMESPACE_KEY, namespace); + parms.put(HTTP_CLIENT_KEY, httpClient); + parms.put(SMART_CLIENT_KEY, smartClient); + logger.debug("using storage config: {}", parms); + + String[] hostArr = endpoint.split(","); + for (int i = 0; i < hostArr.length; i++) { + int stInd = hostArr[i].indexOf("/") + 2; //Starting index of endpoint IP + hostArr[i] = hostArr[i].substring(stInd, hostArr[i].indexOf(":", stInd + 1)); + } + if ( smartClient == Boolean.FALSE ) try { + s3CliConfig = new S3Config(new URI(hostArr[0])); + } catch (URISyntaxException e) { + logger.error(e.getMessage()); + } + else { + try { + if (endpoint.toLowerCase().contains("https")) s3CliConfig = new S3Config(Protocol.HTTPS, hostArr); + else s3CliConfig = new S3Config(Protocol.HTTP, hostArr); + } catch (Exception e) { + logger.error(e.getMessage()); + s3CliConfig = new S3Config(Protocol.HTTPS, hostArr); + } + } + s3CliConfig.withIdentity(accessKey).withSecretKey(secretKey); + + + try { + System.setProperty("http.maxConnections", Integer.toString(Thread.activeCount()) + 128); + if(!connTimeout.equals("")) System.setProperty("com.sun.jersey.client.property.readTimeout", connTimeout); + if(!readTimeout.equals("")) System.setProperty("com.sun.jersey.client.property.connectTimeout", readTimeout); + } + catch (Exception e) { + logger.error(e.getMessage()); + //throw new StorageException(e); + } + + if (!namespace.equals("")) s3CliConfig.setNamespace(namespace); + + if (httpClient == "java") { + s3Client = new S3JerseyClient(s3CliConfig, new URLConnectionClientHandler()); + } + else{ + s3Client = new S3JerseyClient(s3CliConfig); + } + + } + + /** + * Set auth context for this storage API. + * + * @param info AuthContext for this storage API. + */ + public void setAuthContext(AuthContext info) { + super.setAuthContext(info); + } + + /** + * Release held client resource. + */ + public void dispose() { + super.dispose(); + s3Client.destroy(); + s3Client = null; + } + + + /** + * Retrieves an object from a ECS bucket as an InputStream. + * + * @param container bucket containing the desired object. + * @param object key (identifier) of object to be returned + * @param config configuration used for this operation + * @return inputStream an InputStream representing the content + * of the object; if null, the object was not found + */ + public InputStream getObject(String container, String object, Config config) { + + super.getObject(container, object, config); + InputStream stream; + + try { + logger.info((new StringBuilder("Retrieving ")).append(container).append("\\").append(object).toString()); + stream = s3Client.readObjectStream(container, object, new Range(new Long(0), s3Client.getObjectMetadata(container, object).getContentLength())); + } catch (Exception e) { + logger.error(e.getMessage()); + stream = null; + //throw new StorageException(e); + } + return stream; + } + + /** + * Creates a bucket using the ECS client + * + * @param container name of bucket to be created + * @param config configuration used for this operation + */ + public void createContainer(String container, Config config) { + + super.createContainer(container, config); + + try { + if (!s3Client.bucketExists(container)) { + logger.info((new StringBuilder("Creating ")).append(container).toString()); + s3Client.createBucket(container); + } + } catch (Exception e) { + logger.error(e.getMessage()); + //throw new StorageException(e); + } + } + + /** + * Creates an object using the ECS client + * + * @param container name of bucket containing the new object + * @param object key (identifier) of object to be created + * @param data content of the new object + * @param length length of the new object; stored in metadata + * @param config configuration used for this operation + */ + public void createObject(String container, String object, InputStream data, long length, Config config) { + + super.createObject(container, object, data, length, config); + + try { + logger.info((new StringBuilder("Creating ")).append(container).append("\\").append(object).append(" with length=").append(length).append(" Bytes").toString()); + PutObjectRequest req = new PutObjectRequest(container, object, data).withObjectMetadata(new S3ObjectMetadata().withContentLength(length)); + s3Client.putObject(req); + } catch (Exception e) { + logger.error(e.getMessage()); + //throw new StorageException(e); + } + } + + /** + * Deletes a bucket using the ECS client + * + * @param container name of bucket to be deleted + * @param config configuration used for this operation + */ + public void deleteContainer(String container, Config config) { + + super.deleteContainer(container, config); + + try { + if (s3Client.bucketExists(container)) { + logger.info((new StringBuilder("Deleting ")).append(container).toString()); + s3Client.deleteBucket(container); + } + } catch (Exception e) { + logger.error(e.getMessage()); + //throw new StorageException(e); + } + } + + /** + * Deletes an object using the ECS client + * + * @param container name of bucket containing object + * @param object key (identifier) of object to be deleted + * @param config configuration used for this operation + */ + public void deleteObject(String container, String object, Config config) { + + super.deleteObject(container, object, config); + + try { + logger.info((new StringBuilder("Deleting ")).append(container).append("\\").append(object).toString()); + s3Client.deleteObject(container, object); + } catch (Exception e) { + + logger.error(e.getMessage()); + //throw new StorageException(e); + } + } + +} \ No newline at end of file diff --git a/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/ECSStorageFactory.java b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/ECSStorageFactory.java new file mode 100644 index 00000000..adc57aa0 --- /dev/null +++ b/dev/cosbench-ecs/src/com/emc/vipr/cosbench/ECSStor/ECSStorageFactory.java @@ -0,0 +1,40 @@ +/* + * Copyright 2014-2016 EMC Corporation. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0.txt + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.emc.vipr.cosbench.ECSStor; + +import com.intel.cosbench.api.storage.StorageAPI; +import com.intel.cosbench.api.storage.StorageAPIFactory; + +/** + * Factory class for EMC ECS Storage API plugin for + * COSBench. + */ +public class ECSStorageFactory implements StorageAPIFactory { + + /** + * Returnes identifying name of this Storage API instance + */ + public String getStorageName() { + return "ecs"; + } + + /** + * Returns a new instance of this Storage API + */ + public StorageAPI getStorageAPI() { + return new ECSStorage(); + } +} \ No newline at end of file diff --git a/dev/cosbench-ecs/vipr-object-transformations-2.0.3.jar b/dev/cosbench-ecs/vipr-object-transformations-2.0.3.jar new file mode 100644 index 00000000..5091ab49 Binary files /dev/null and b/dev/cosbench-ecs/vipr-object-transformations-2.0.3.jar differ diff --git a/dev/cosbench-gcs/.classpath b/dev/cosbench-gcs/.classpath new file mode 100644 index 00000000..86599bcd --- /dev/null +++ b/dev/cosbench-gcs/.classpath @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/dev/cosbench-gcs/.project b/dev/cosbench-gcs/.project new file mode 100644 index 00000000..dd26db51 --- /dev/null +++ b/dev/cosbench-gcs/.project @@ -0,0 +1,28 @@ + + + cosbench-gcs + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/dev/cosbench-gcs/META-INF/MANIFEST.MF b/dev/cosbench-gcs/META-INF/MANIFEST.MF new file mode 100644 index 00000000..558f8d85 --- /dev/null +++ b/dev/cosbench-gcs/META-INF/MANIFEST.MF @@ -0,0 +1,37 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Google Cloud Storage Client Bundle +Bundle-SymbolicName: cosbench-gcs +Bundle-Version: 0.4.1.0 +Bundle-Vendor: Intel +Bundle-RequiredExecutionEnvironment: JavaSE-1.6 +Import-Package: com.intel.cosbench.api.auth, + com.intel.cosbench.api.context, + com.intel.cosbench.api.storage, + com.intel.cosbench.client.http, + com.intel.cosbench.config, + com.intel.cosbench.log, + org.apache.commons.codec;version="[1.3.0,2.0.0)", + org.apache.commons.codec.net;version="[1.3.0,2.0.0)", + org.apache.http;version="[4.1.4,5.0.0)", + org.apache.http.client;version="[4.1.3,5.0.0)", + org.apache.http.client.methods;version="[4.1.3,5.0.0)", + org.apache.http.conn;version="[4.1.3,5.0.0)", + org.apache.http.entity;version="[4.1.4,5.0.0)", + org.apache.http.message;version="[4.1.4,5.0.0)", + org.apache.http.params;version="[4.1.4,5.0.0)", + org.apache.http.util;version="[4.1.4,5.0.0)" +Bundle-ClassPath: ., + google-api-client-1.19.0.jar, + google-api-services-storage-v1-rev18-1.19.0.jar, + google-http-client-1.19.0.jar, + google-http-client-jackson2-1.19.0.jar, + google-oauth-client-1.19.0.jar, + commons-codec-1.3.jar, + commons-logging-1.1.1.jar, + guava-jdk5-13.0.jar, + hamcrest-core-1.1.jar, + httpclient-4.0.1.jar, + httpcore-4.0.1.jar, + jackson-core-2.1.3.jar, + jsr305-1.3.9.jar diff --git a/dev/cosbench-gcs/META-INF/spring/plugin-context.xml b/dev/cosbench-gcs/META-INF/spring/plugin-context.xml new file mode 100644 index 00000000..562d293c --- /dev/null +++ b/dev/cosbench-gcs/META-INF/spring/plugin-context.xml @@ -0,0 +1,17 @@ + + + + + + + + + + \ No newline at end of file diff --git a/dev/cosbench-gcs/build.properties b/dev/cosbench-gcs/build.properties new file mode 100644 index 00000000..cca85127 --- /dev/null +++ b/dev/cosbench-gcs/build.properties @@ -0,0 +1,18 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + .,\ + google-api-client-1.19.0.jar,\ + google-api-services-storage-v1-rev18-1.19.0.jar,\ + google-http-client-1.19.0.jar,\ + google-http-client-jackson2-1.19.0.jar,\ + google-oauth-client-1.19.0.jar,\ + guava-jdk5-13.0.jar,\ + hamcrest-core-1.1.jar,\ + httpclient-4.0.1.jar,\ + httpcore-4.0.1.jar,\ + jackson-core-2.1.3.jar,\ + jsr305-1.3.9.jar,\ + commons-codec-1.3.jar,\ + commons-logging-1.1.1.jar + diff --git a/dev/cosbench-s3/commons-codec-1.3.jar b/dev/cosbench-gcs/commons-codec-1.3.jar similarity index 100% rename from dev/cosbench-s3/commons-codec-1.3.jar rename to dev/cosbench-gcs/commons-codec-1.3.jar diff --git a/dev/cosbench-gcs/commons-logging-1.1.1.jar b/dev/cosbench-gcs/commons-logging-1.1.1.jar new file mode 100644 index 00000000..1deef144 Binary files /dev/null and b/dev/cosbench-gcs/commons-logging-1.1.1.jar differ diff --git a/dev/cosbench-gcs/google-api-client-1.19.0.jar b/dev/cosbench-gcs/google-api-client-1.19.0.jar new file mode 100644 index 00000000..b641cb73 Binary files /dev/null and b/dev/cosbench-gcs/google-api-client-1.19.0.jar differ diff --git a/dev/cosbench-gcs/google-api-services-storage-v1-rev18-1.19.0.jar b/dev/cosbench-gcs/google-api-services-storage-v1-rev18-1.19.0.jar new file mode 100644 index 00000000..fc667f73 Binary files /dev/null and b/dev/cosbench-gcs/google-api-services-storage-v1-rev18-1.19.0.jar differ diff --git a/dev/cosbench-gcs/google-http-client-1.19.0.jar b/dev/cosbench-gcs/google-http-client-1.19.0.jar new file mode 100644 index 00000000..3204499a Binary files /dev/null and b/dev/cosbench-gcs/google-http-client-1.19.0.jar differ diff --git a/dev/cosbench-gcs/google-http-client-jackson2-1.19.0.jar b/dev/cosbench-gcs/google-http-client-jackson2-1.19.0.jar new file mode 100644 index 00000000..f60d0977 Binary files /dev/null and b/dev/cosbench-gcs/google-http-client-jackson2-1.19.0.jar differ diff --git a/dev/cosbench-gcs/google-oauth-client-1.19.0.jar b/dev/cosbench-gcs/google-oauth-client-1.19.0.jar new file mode 100644 index 00000000..db4837e9 Binary files /dev/null and b/dev/cosbench-gcs/google-oauth-client-1.19.0.jar differ diff --git a/dev/cosbench-gcs/guava-jdk5-13.0.jar b/dev/cosbench-gcs/guava-jdk5-13.0.jar new file mode 100644 index 00000000..85881e5d Binary files /dev/null and b/dev/cosbench-gcs/guava-jdk5-13.0.jar differ diff --git a/dev/cosbench-gcs/hamcrest-core-1.1.jar b/dev/cosbench-gcs/hamcrest-core-1.1.jar new file mode 100644 index 00000000..e5149be7 Binary files /dev/null and b/dev/cosbench-gcs/hamcrest-core-1.1.jar differ diff --git a/dev/cosbench-gcs/httpclient-4.0.1.jar b/dev/cosbench-gcs/httpclient-4.0.1.jar new file mode 100644 index 00000000..ad32285a Binary files /dev/null and b/dev/cosbench-gcs/httpclient-4.0.1.jar differ diff --git a/dev/cosbench-gcs/httpcore-4.0.1.jar b/dev/cosbench-gcs/httpcore-4.0.1.jar new file mode 100644 index 00000000..4aef35e2 Binary files /dev/null and b/dev/cosbench-gcs/httpcore-4.0.1.jar differ diff --git a/dev/cosbench-gcs/jackson-core-2.1.3.jar b/dev/cosbench-gcs/jackson-core-2.1.3.jar new file mode 100644 index 00000000..f47619f4 Binary files /dev/null and b/dev/cosbench-gcs/jackson-core-2.1.3.jar differ diff --git a/dev/cosbench-gcs/jsr305-1.3.9.jar b/dev/cosbench-gcs/jsr305-1.3.9.jar new file mode 100644 index 00000000..a9afc661 Binary files /dev/null and b/dev/cosbench-gcs/jsr305-1.3.9.jar differ diff --git a/dev/cosbench-gcs/src/com/intel/cosbench/api/GcsStor/GcsStorage.java b/dev/cosbench-gcs/src/com/intel/cosbench/api/GcsStor/GcsStorage.java new file mode 100644 index 00000000..5d61edda --- /dev/null +++ b/dev/cosbench-gcs/src/com/intel/cosbench/api/GcsStor/GcsStorage.java @@ -0,0 +1,147 @@ +package com.intel.cosbench.api.GcsStor; + +import static com.intel.cosbench.client.GcsStor.GcsStorConstants.CONN_TIMEOUT_KEY; +import static com.intel.cosbench.client.GcsStor.GcsStorConstants.CONN_TIMEOUT_DEFAULT; +import static com.intel.cosbench.client.GcsStor.GcsStorConstants.JSON_KEY_FILE; +import static com.intel.cosbench.client.GcsStor.GcsStorConstants.JSON_KEY_FILE_DEFAULT; +import static com.intel.cosbench.client.GcsStor.GcsStorConstants.PROJECT_ID; +import static com.intel.cosbench.client.GcsStor.GcsStorConstants.PROJECT_ID_DEFAULT; + +import java.io.*; + +import com.intel.cosbench.api.storage.*; +import com.intel.cosbench.api.context.*; +import com.intel.cosbench.config.Config; +import com.intel.cosbench.log.Logger; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.InputStreamContent; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.storage.Storage; +import com.google.api.services.storage.StorageScopes; +import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.StorageObject; + +public class GcsStorage extends NoneStorage { + private int timeout; + private String jsonKeyFile; + private String projectId; + private static Storage client; + + private static final JsonFactory JSON_FACTORY = JacksonFactory + .getDefaultInstance(); + + @Override + public void init(Config config, Logger logger) { + super.init(config, logger); + initParms(config); + try { + InputStream input = new FileInputStream(jsonKeyFile); + GoogleCredential credential = GoogleCredential.fromStream(input); + if (credential.createScopedRequired()) { + credential = credential.createScoped(StorageScopes.all()); + } + HttpTransport httpTransport = GoogleNetHttpTransport + .newTrustedTransport(); + client = new Storage.Builder(httpTransport, JSON_FACTORY, + credential).build(); + logger.debug("GCS client has been initialized"); + } catch (Exception e) { + throw new StorageException(e); + } + } + + private void initParms(Config config) { + timeout = config.getInt(CONN_TIMEOUT_KEY, CONN_TIMEOUT_DEFAULT); + parms.put(CONN_TIMEOUT_KEY, timeout); + + jsonKeyFile = config.get(JSON_KEY_FILE, JSON_KEY_FILE_DEFAULT); + parms.put(JSON_KEY_FILE, jsonKeyFile); + + projectId = config.get(PROJECT_ID, PROJECT_ID_DEFAULT); + parms.put(PROJECT_ID, projectId); + + logger.debug("using storage config: {}", parms); + } + + @Override + public void setAuthContext(AuthContext info) { + super.setAuthContext(info); + } + + @Override + public void dispose() { + super.dispose(); + client = null; + } + + @Override + public InputStream getObject(String container, String object, Config config) { + super.getObject(container, object, config); + InputStream stream; + try { + Storage.Objects.Get getRequest = client.objects().get(container, + object); + stream = getRequest.executeMediaAsInputStream(); + } catch (IOException e) { + throw new StorageException(e); + } + return stream; + } + + @Override + public void createContainer(String container, Config config) { + super.createContainer(container, config); + Bucket newBucket = new Bucket(); + newBucket.setName(container); + try { + Storage.Buckets.Insert bucketInsertRequest = client.buckets() + .insert(projectId, newBucket); + bucketInsertRequest.execute(); + } catch (IOException e) { + throw new StorageException(e); + } + } + + @Override + public void createObject(String container, String object, InputStream data, + long length, Config config) { + super.createObject(container, object, data, length, config); + InputStreamContent contentStream = new InputStreamContent( + "application/octet-stream", data); + StorageObject objectMetadata = new StorageObject().setName(object); + try { + Storage.Objects.Insert insertRequest = client.objects().insert( + container, objectMetadata, contentStream); + insertRequest.execute(); + } catch (IOException e) { + throw new StorageException(e); + } + + } + + @Override + public void deleteContainer(String container, Config config) { + super.deleteContainer(container, config); + try { + Storage.Buckets.Delete bucketDeleteRequest = client.buckets() + .delete(container); + bucketDeleteRequest.execute(); + } catch (IOException e) { + throw new StorageException(e); + } + } + + @Override + public void deleteObject(String container, String object, Config config) { + super.deleteObject(container, object, config); + try { + client.objects().delete(container, object).execute(); + } catch (Exception e) { + throw new StorageException(e); + } + } + +} diff --git a/dev/cosbench-gcs/src/com/intel/cosbench/api/GcsStor/GcsStorageFactory.java b/dev/cosbench-gcs/src/com/intel/cosbench/api/GcsStor/GcsStorageFactory.java new file mode 100644 index 00000000..000d3e78 --- /dev/null +++ b/dev/cosbench-gcs/src/com/intel/cosbench/api/GcsStor/GcsStorageFactory.java @@ -0,0 +1,17 @@ +package com.intel.cosbench.api.GcsStor; + +import com.intel.cosbench.api.storage.*; + +public class GcsStorageFactory implements StorageAPIFactory { + + @Override + public String getStorageName() { + return "gcs"; + } + + @Override + public StorageAPI getStorageAPI() { + return new GcsStorage(); + } + +} diff --git a/dev/cosbench-gcs/src/com/intel/cosbench/client/GcsStor/GcsStorConstants.java b/dev/cosbench-gcs/src/com/intel/cosbench/client/GcsStor/GcsStorConstants.java new file mode 100644 index 00000000..954d41e7 --- /dev/null +++ b/dev/cosbench-gcs/src/com/intel/cosbench/client/GcsStor/GcsStorConstants.java @@ -0,0 +1,24 @@ +package com.intel.cosbench.client.GcsStor; + +public interface GcsStorConstants { + + // -------------------------------------------------------------------------- + // CONNECTION + // -------------------------------------------------------------------------- + + String CONN_TIMEOUT_KEY = "timeout"; + int CONN_TIMEOUT_DEFAULT = 30000; + + // -------------------------------------------------------------------------- + // PROJECT_ID + // -------------------------------------------------------------------------- + String PROJECT_ID = "projectid"; + String PROJECT_ID_DEFAULT = "xxxxx-xxxx"; + + // -------------------------------------------------------------------------- + // JSON_KEY_FILE + // -------------------------------------------------------------------------- + String JSON_KEY_FILE = "jsonkeyfile"; + String JSON_KEY_FILE_DEFAULT = "/path/to/json/key/file"; + +} diff --git a/dev/cosbench-openio/.classpath b/dev/cosbench-openio/.classpath new file mode 100644 index 00000000..72649e55 --- /dev/null +++ b/dev/cosbench-openio/.classpath @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/dev/cosbench-openio/.project b/dev/cosbench-openio/.project new file mode 100644 index 00000000..eee67ecc --- /dev/null +++ b/dev/cosbench-openio/.project @@ -0,0 +1,28 @@ + + + cosbench-openio + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.pde.PluginNature + + diff --git a/dev/cosbench-openio/.settings/org.eclipse.jdt.core.prefs b/dev/cosbench-openio/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000..8000cd6c --- /dev/null +++ b/dev/cosbench-openio/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,11 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.6 diff --git a/dev/cosbench-openio/META-INF/MANIFEST.MF b/dev/cosbench-openio/META-INF/MANIFEST.MF new file mode 100644 index 00000000..5e5431fa --- /dev/null +++ b/dev/cosbench-openio/META-INF/MANIFEST.MF @@ -0,0 +1,27 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: OpenIO Storage Client Bundle +Bundle-SymbolicName: cosbench-openio +Bundle-Version: 0.4.1.0 +Bundle-Vendor: Intel +Bundle-RequiredExecutionEnvironment: JavaSE-1.6 +Import-Package: com.intel.cosbench.api.auth, + com.intel.cosbench.api.context, + com.intel.cosbench.api.storage, + com.intel.cosbench.client.http, + com.intel.cosbench.config, + com.intel.cosbench.log, + org.apache.commons.codec;version="[1.3.0,2.0.0)", + org.apache.commons.codec.net;version="[1.3.0,2.0.0)", + org.apache.http;version="[4.1.4,5.0.0)", + org.apache.http.client;version="[4.1.3,5.0.0)", + org.apache.http.client.methods;version="[4.1.3,5.0.0)", + org.apache.http.conn;version="[4.1.3,5.0.0)", + org.apache.http.entity;version="[4.1.4,5.0.0)", + org.apache.http.message;version="[4.1.4,5.0.0)", + org.apache.http.params;version="[4.1.4,5.0.0)", + org.apache.http.util;version="[4.1.4,5.0.0)" +Bundle-ClassPath: ., + gson-2.5.jar, + openio-api-0.3.2-SNAPSHOT.jar +Export-Package: com.intel.cosbench.api.openio diff --git a/dev/cosbench-openio/META-INF/spring/plugin-context.xml b/dev/cosbench-openio/META-INF/spring/plugin-context.xml new file mode 100644 index 00000000..0d8e7973 --- /dev/null +++ b/dev/cosbench-openio/META-INF/spring/plugin-context.xml @@ -0,0 +1,17 @@ + + + + + + + + + + \ No newline at end of file diff --git a/dev/cosbench-openio/REAME.md b/dev/cosbench-openio/REAME.md new file mode 100644 index 00000000..a6fd015d --- /dev/null +++ b/dev/cosbench-openio/REAME.md @@ -0,0 +1,8 @@ +

OpenIO COSBench plugin

+ +

Required elements to set in Config object: +

+

The plugin implements basis storage functions (create/delete containers and create / get / delete objects).

\ No newline at end of file diff --git a/dev/cosbench-openio/build.properties b/dev/cosbench-openio/build.properties new file mode 100644 index 00000000..6c2ae212 --- /dev/null +++ b/dev/cosbench-openio/build.properties @@ -0,0 +1,6 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + .,\ + gson-2.5.jar,\ + openio-api-0.3.2-SNAPSHOT.jar diff --git a/dev/cosbench-openio/gson-2.5.jar b/dev/cosbench-openio/gson-2.5.jar new file mode 100644 index 00000000..5c35c5d5 Binary files /dev/null and b/dev/cosbench-openio/gson-2.5.jar differ diff --git a/dev/cosbench-openio/openio-api-0.3.2-SNAPSHOT.jar b/dev/cosbench-openio/openio-api-0.3.2-SNAPSHOT.jar new file mode 100644 index 00000000..0cbf5dcd Binary files /dev/null and b/dev/cosbench-openio/openio-api-0.3.2-SNAPSHOT.jar differ diff --git a/dev/cosbench-openio/src/com/intel/cosbench/api/openio/OpenIOStorage.java b/dev/cosbench-openio/src/com/intel/cosbench/api/openio/OpenIOStorage.java new file mode 100644 index 00000000..dc318458 --- /dev/null +++ b/dev/cosbench-openio/src/com/intel/cosbench/api/openio/OpenIOStorage.java @@ -0,0 +1,145 @@ +package com.intel.cosbench.api.openio; + +import static com.intel.cosbench.client.openio.OioStorageConstants.ACCOUNT_KEY; +import static com.intel.cosbench.client.openio.OioStorageConstants.NS_KEY; +import static com.intel.cosbench.client.openio.OioStorageConstants.PROXYD_URL_KEY; +import static io.openio.sds.models.OioUrl.url; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.intel.cosbench.api.storage.NoneStorage; +import com.intel.cosbench.api.storage.StorageException; +import com.intel.cosbench.config.Config; +import com.intel.cosbench.log.Logger; + +import io.openio.sds.Client; +import io.openio.sds.ClientBuilder; +import io.openio.sds.exceptions.ContainerExistException; +import io.openio.sds.exceptions.ContainerNotEmptyException; +import io.openio.sds.exceptions.SdsException; +import io.openio.sds.models.ListOptions; +import io.openio.sds.models.ObjectInfo; +import io.openio.sds.models.ObjectList; +import io.openio.sds.models.ObjectList.ObjectView; + +/** + * + * @author Christopher Dedeurwaerder + * + */ +public class OpenIOStorage extends NoneStorage { + + private Client client; + private String account; + private Pattern errCodePattern = Pattern.compile(".*(\\p{Digit}{3}).*"); + + @Override + public void init(Config config, Logger logger) { + super.init(config, logger); + String ns = config.get(NS_KEY); + String proxydurl = config.get(PROXYD_URL_KEY); + client = ClientBuilder + .newClient(ns, proxydurl); + account = config.get(ACCOUNT_KEY); + parms.put(NS_KEY, ns); + parms.put(PROXYD_URL_KEY, proxydurl); + parms.put(ACCOUNT_KEY, account); + } + + /** + * Hack the exception message to put the error code where COSBench expects + * it. + * + * @param wrapped + * The exception to wrap + * @return a new StorageException with the {@link SdsException} as cause + */ + private StorageException makeStorageException(SdsException wrapped) { + String origMsg = wrapped.getMessage(); + if (wrapped instanceof ContainerNotEmptyException + || wrapped instanceof ContainerExistException) + return new StorageException("HTTP/1.1 409 " + origMsg, wrapped); + + Matcher codeMatcher = errCodePattern.matcher(origMsg); + if (codeMatcher.find()) { + return new StorageException("HTTP/1.1 " + codeMatcher.group(1) + + " " + origMsg, wrapped); + } + return new StorageException("HTTP/1.1 500 " + origMsg, wrapped); + } + + @Override + public InputStream getObject(String container, String object, + Config config) { + try { + ObjectInfo oinf = client + .getObjectInfo(url(account, container, object)); + return client.downloadObject(oinf); + } catch (SdsException e) { + throw makeStorageException(e); + } + } + + @Override + public InputStream getList(String container, String object, Config config) { + StringBuilder sb = new StringBuilder(); + try { + // TODO: loop on listContainer until the answer is empty + ObjectList objects = client.listContainer(url(account, container), + new ListOptions()); + for (ObjectView obj : objects.objects()) { + sb.append(obj.name()).append('\n'); + } + } catch (SdsException e) { + throw makeStorageException(e); + } + return new ByteArrayInputStream(sb.toString().getBytes()); + } + + @Override + public void createContainer(String container, Config config) { + super.createContainer(container, config); + try { + client.createContainer(url(account, container)); + } catch (ContainerExistException e) { + this.logger.warn("Container " + container + " already exists!", e); + } catch (SdsException e) { + throw makeStorageException(e); + } + } + + @Override + public void createObject(String container, String object, InputStream data, + long length, Config config) { + super.createObject(container, object, data, length, config); + try { + client.putObject(url(account, container, object), + length, data); + } catch (SdsException e) { + throw makeStorageException(e); + } + } + + @Override + public void deleteContainer(String container, Config config) { + super.deleteContainer(container, config); + try { + client.deleteContainer(url(account, container)); + } catch (SdsException e) { + throw makeStorageException(e); + } + } + + @Override + public void deleteObject(String container, String object, Config config) { + super.deleteObject(container, object, config); + try { + client.deleteObject(url(account, container, object)); + } catch (SdsException e) { + throw makeStorageException(e); + } + } +} diff --git a/dev/cosbench-openio/src/com/intel/cosbench/api/openio/OpenIOStorageFactory.java b/dev/cosbench-openio/src/com/intel/cosbench/api/openio/OpenIOStorageFactory.java new file mode 100644 index 00000000..610ea2e3 --- /dev/null +++ b/dev/cosbench-openio/src/com/intel/cosbench/api/openio/OpenIOStorageFactory.java @@ -0,0 +1,25 @@ +package com.intel.cosbench.api.openio; + +import com.intel.cosbench.api.storage.StorageAPI; +import com.intel.cosbench.api.storage.StorageAPIFactory; + +/** + * + * @author Christopher Dedeurwaerder + * + */ +public class OpenIOStorageFactory implements StorageAPIFactory { + + private static final String OIO_STORAGE_NAME = "openio"; + + @Override + public String getStorageName() { + return OIO_STORAGE_NAME; + } + + @Override + public StorageAPI getStorageAPI() { + return new OpenIOStorage(); + } + +} diff --git a/dev/cosbench-openio/src/com/intel/cosbench/client/openio/OioStorageConstants.java b/dev/cosbench-openio/src/com/intel/cosbench/client/openio/OioStorageConstants.java new file mode 100644 index 00000000..c353ddab --- /dev/null +++ b/dev/cosbench-openio/src/com/intel/cosbench/client/openio/OioStorageConstants.java @@ -0,0 +1,15 @@ +package com.intel.cosbench.client.openio; + +/** + * + * @author Christopher Dedeurwaerder + * + */ +public class OioStorageConstants { + + public static final String PROXYD_URL_KEY = "proxyd-url"; + public static final String ACCOUNT_KEY = "account"; + public static final String NS_KEY = "ns"; + + +} diff --git a/dev/cosbench-oss/.classpath b/dev/cosbench-oss/.classpath new file mode 100644 index 00000000..f0fb6aaf --- /dev/null +++ b/dev/cosbench-oss/.classpath @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/dev/cosbench-oss/.project b/dev/cosbench-oss/.project new file mode 100644 index 00000000..6e578002 --- /dev/null +++ b/dev/cosbench-oss/.project @@ -0,0 +1,28 @@ + + + cosbench-oss + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/dev/cosbench-oss/.settings/org.eclipse.jdt.core.prefs b/dev/cosbench-oss/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000..c537b630 --- /dev/null +++ b/dev/cosbench-oss/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,7 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.6 diff --git a/dev/cosbench-oss/.settings/org.eclipse.pde.core.prefs b/dev/cosbench-oss/.settings/org.eclipse.pde.core.prefs new file mode 100644 index 00000000..f29e940a --- /dev/null +++ b/dev/cosbench-oss/.settings/org.eclipse.pde.core.prefs @@ -0,0 +1,3 @@ +eclipse.preferences.version=1 +pluginProject.extensions=false +resolve.requirebundle=false diff --git a/dev/cosbench-oss/META-INF/MANIFEST.MF b/dev/cosbench-oss/META-INF/MANIFEST.MF new file mode 100644 index 00000000..7daf58be --- /dev/null +++ b/dev/cosbench-oss/META-INF/MANIFEST.MF @@ -0,0 +1,21 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Cosbench-oss +Bundle-SymbolicName: cosbench-oss +Bundle-Version: 0.4.2.0 +Bundle-Vendor: intel +Bundle-RequiredExecutionEnvironment: JavaSE-1.6 +Bundle-ClassPath: ., + aliyun-sdk-oss-2.8.1.jar, + httpcore-4.4.jar, + httpclient-4.4.jar, + commons-logging-1.2.jar, + commons-codec-1.9.jar, + hamcrest-core-1.1.jar, + jdom-1.1.jar +Import-Package: com.intel.cosbench.api.auth, + com.intel.cosbench.api.context, + com.intel.cosbench.api.storage, + com.intel.cosbench.client.http, + com.intel.cosbench.config, + com.intel.cosbench.log diff --git a/dev/cosbench-oss/META-INF/spring/plugin-context.xml b/dev/cosbench-oss/META-INF/spring/plugin-context.xml new file mode 100644 index 00000000..ef4c16be --- /dev/null +++ b/dev/cosbench-oss/META-INF/spring/plugin-context.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/dev/cosbench-oss/aliyun-sdk-oss-2.8.1.jar b/dev/cosbench-oss/aliyun-sdk-oss-2.8.1.jar new file mode 100644 index 00000000..4cd5bd15 Binary files /dev/null and b/dev/cosbench-oss/aliyun-sdk-oss-2.8.1.jar differ diff --git a/dev/cosbench-oss/build.properties b/dev/cosbench-oss/build.properties new file mode 100644 index 00000000..a18011b0 --- /dev/null +++ b/dev/cosbench-oss/build.properties @@ -0,0 +1,11 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + .,\ + aliyun-sdk-oss-2.8.1.jar,\ + httpcore-4.4.jar,\ + httpclient-4.4.jar,\ + commons-logging-1.2.jar,\ + commons-codec-1.9.jar,\ + hamcrest-core-1.1.jar,\ + jdom-1.1.jar diff --git a/dev/cosbench-oss/commons-codec-1.9.jar b/dev/cosbench-oss/commons-codec-1.9.jar new file mode 100644 index 00000000..ef35f1c5 Binary files /dev/null and b/dev/cosbench-oss/commons-codec-1.9.jar differ diff --git a/dev/cosbench-oss/commons-logging-1.2.jar b/dev/cosbench-oss/commons-logging-1.2.jar new file mode 100644 index 00000000..93a3b9f6 Binary files /dev/null and b/dev/cosbench-oss/commons-logging-1.2.jar differ diff --git a/dev/cosbench-oss/hamcrest-core-1.1.jar b/dev/cosbench-oss/hamcrest-core-1.1.jar new file mode 100644 index 00000000..e5149be7 Binary files /dev/null and b/dev/cosbench-oss/hamcrest-core-1.1.jar differ diff --git a/dev/cosbench-oss/httpclient-4.4.jar b/dev/cosbench-oss/httpclient-4.4.jar new file mode 100644 index 00000000..998e3d5b Binary files /dev/null and b/dev/cosbench-oss/httpclient-4.4.jar differ diff --git a/dev/cosbench-oss/httpcore-4.4.jar b/dev/cosbench-oss/httpcore-4.4.jar new file mode 100644 index 00000000..27a10480 Binary files /dev/null and b/dev/cosbench-oss/httpcore-4.4.jar differ diff --git a/dev/cosbench-oss/jdom-1.1.jar b/dev/cosbench-oss/jdom-1.1.jar new file mode 100644 index 00000000..97c85f56 Binary files /dev/null and b/dev/cosbench-oss/jdom-1.1.jar differ diff --git a/dev/cosbench-oss/src/com/intel/cosbench/api/oss/OSSStorage.java b/dev/cosbench-oss/src/com/intel/cosbench/api/oss/OSSStorage.java new file mode 100644 index 00000000..e81a44c5 --- /dev/null +++ b/dev/cosbench-oss/src/com/intel/cosbench/api/oss/OSSStorage.java @@ -0,0 +1,171 @@ +package com.intel.cosbench.api.oss; + +import static com.intel.cosbench.client.oss.OSSConstants.AUTH_PASSWORD_DEFAULT; +import static com.intel.cosbench.client.oss.OSSConstants.AUTH_PASSWORD_KEY; +import static com.intel.cosbench.client.oss.OSSConstants.AUTH_USERNAME_DEFAULT; +import static com.intel.cosbench.client.oss.OSSConstants.AUTH_USERNAME_KEY; +import static com.intel.cosbench.client.oss.OSSConstants.CONN_TIMEOUT_DEFAULT; +import static com.intel.cosbench.client.oss.OSSConstants.CONN_TIMEOUT_KEY; +import static com.intel.cosbench.client.oss.OSSConstants.ENDPOINT_DEFAULT; +import static com.intel.cosbench.client.oss.OSSConstants.ENDPOINT_KEY; +import static com.intel.cosbench.client.oss.OSSConstants.PATH_STYLE_ACCESS_DEFAULT; +import static com.intel.cosbench.client.oss.OSSConstants.PATH_STYLE_ACCESS_KEY; +import static com.intel.cosbench.client.oss.OSSConstants.PROXY_HOST_KEY; +import static com.intel.cosbench.client.oss.OSSConstants.PROXY_PORT_KEY; + +import java.io.InputStream; + +import com.aliyun.oss.ClientConfiguration; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.common.auth.Credentials; +import com.aliyun.oss.common.auth.CredentialsProvider; +import com.aliyun.oss.common.auth.DefaultCredentialProvider; +import com.aliyun.oss.common.auth.DefaultCredentials; +import com.aliyun.oss.model.OSSObject; +import com.aliyun.oss.model.ObjectMetadata; +import com.intel.cosbench.api.context.AuthContext; +import com.intel.cosbench.api.context.Context; +import com.intel.cosbench.api.storage.NoneStorage; +import com.intel.cosbench.api.storage.StorageException; +import com.intel.cosbench.config.Config; +import com.intel.cosbench.log.Logger; + +public class OSSStorage extends NoneStorage { + + private int timeout; + private String accessKey; + private String secretKey; + private String endpoint; + + private OSS ossClient; + + @Override + public void init(Config config, Logger logger) { + super.init(config, logger); + + timeout = config.getInt(CONN_TIMEOUT_KEY, CONN_TIMEOUT_DEFAULT); + + parms.put(CONN_TIMEOUT_KEY, timeout); + + endpoint = config.get(ENDPOINT_KEY, ENDPOINT_DEFAULT); + accessKey = config.get(AUTH_USERNAME_KEY, AUTH_USERNAME_DEFAULT); + secretKey = config.get(AUTH_PASSWORD_KEY, AUTH_PASSWORD_DEFAULT); + + boolean pathStyleAccess = config.getBoolean(PATH_STYLE_ACCESS_KEY, PATH_STYLE_ACCESS_DEFAULT); + + String proxyHost = config.get(PROXY_HOST_KEY, ""); + String proxyPort = config.get(PROXY_PORT_KEY, ""); + + parms.put(ENDPOINT_KEY, endpoint); + parms.put(AUTH_USERNAME_KEY, accessKey); + parms.put(AUTH_PASSWORD_KEY, secretKey); + parms.put(PATH_STYLE_ACCESS_KEY, pathStyleAccess); + parms.put(PROXY_HOST_KEY, proxyHost); + parms.put(PROXY_PORT_KEY, proxyPort); + + logger.debug("using storage config: {}", parms); + + ClientConfiguration clientConf = new ClientConfiguration(); + clientConf.setConnectionTimeout(timeout); + clientConf.setSocketTimeout(timeout); + // clientConf.setProtocol(Protocol.HTTP); + if ((!proxyHost.equals("")) && (!proxyPort.equals(""))) { + clientConf.setProxyHost(proxyHost); + clientConf.setProxyPort(Integer.parseInt(proxyPort)); + } + + Credentials ossCredentials = new DefaultCredentials(accessKey, secretKey); + CredentialsProvider credsProvider = new DefaultCredentialProvider(ossCredentials); + ossClient = new OSSClient(endpoint, credsProvider, clientConf); + + logger.debug("aliyun oss client has been initialized"); + } + + @Override + public void setAuthContext(AuthContext info) { + super.setAuthContext(info); + } + + @Override + public void dispose() { + super.dispose(); + ossClient = null; + } + + @Override + public Context getParms() { + return super.getParms(); + } + + @Override + public InputStream getObject(String container, String object, Config config) { + super.getObject(container, object, config); + InputStream stream; + try { + OSSObject ossObject = ossClient.getObject(container, object); + stream = ossObject.getObjectContent(); + } catch (OSSException ossExce) { + throw new StorageException("error message:" + ossExce.getErrorMessage(), ossExce); + } catch (Exception e) { + throw new StorageException(e); + } + return stream; + } + + @Override + public void createContainer(String container, Config config) { + super.createContainer(container, config); + try { + if (!ossClient.doesBucketExist(container)) { + ossClient.createBucket(container); + } + } catch (OSSException ossExce) { + throw new StorageException(ossExce.getErrorMessage(), ossExce); + } catch (Exception e) { + throw new StorageException(e); + } + } + + @Override + public void createObject(String container, String object, InputStream data, long length, Config config) { + super.createObject(container, object, data, length, config); + try { + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(length); + metadata.setContentType("application/octet-stream"); + ossClient.putObject(container, object, data, metadata); + } catch (OSSException ossExce) { + throw new StorageException(ossExce.getErrorMessage(), ossExce); + } catch (Exception e) { + throw new StorageException(e); + } + } + + @Override + public void deleteContainer(String container, Config config) { + super.deleteContainer(container, config); + try { + if (ossClient.doesBucketExist(container)) { + ossClient.deleteBucket(container); + } + } catch (OSSException ossExce) { + throw new StorageException(ossExce.getErrorMessage(), ossExce); + } catch (Exception e) { + throw new StorageException(e); + } + } + + @Override + public void deleteObject(String container, String object, Config config) { + super.deleteObject(container, object, config); + try { + ossClient.deleteObject(container, object); + } catch (OSSException ossExce) { + throw new StorageException(ossExce.getErrorMessage(), ossExce); + } catch (Exception e) { + throw new StorageException(e); + } + } +} diff --git a/dev/cosbench-oss/src/com/intel/cosbench/api/oss/OSSStorageFactory.java b/dev/cosbench-oss/src/com/intel/cosbench/api/oss/OSSStorageFactory.java new file mode 100644 index 00000000..43831719 --- /dev/null +++ b/dev/cosbench-oss/src/com/intel/cosbench/api/oss/OSSStorageFactory.java @@ -0,0 +1,18 @@ +package com.intel.cosbench.api.oss; + +import com.intel.cosbench.api.storage.StorageAPI; +import com.intel.cosbench.api.storage.StorageAPIFactory; + +public class OSSStorageFactory implements StorageAPIFactory { + + @Override + public String getStorageName() { + return "oss"; + } + + @Override + public StorageAPI getStorageAPI() { + return new OSSStorage(); + } + +} diff --git a/dev/cosbench-oss/src/com/intel/cosbench/client/oss/OSSConstants.java b/dev/cosbench-oss/src/com/intel/cosbench/client/oss/OSSConstants.java new file mode 100644 index 00000000..943cea8a --- /dev/null +++ b/dev/cosbench-oss/src/com/intel/cosbench/client/oss/OSSConstants.java @@ -0,0 +1,44 @@ +package com.intel.cosbench.client.oss; + +public interface OSSConstants { + + // -------------------------------------------------------------------------- + // CONNECTION + // -------------------------------------------------------------------------- + + String CONN_TIMEOUT_KEY = "timeout"; + int CONN_TIMEOUT_DEFAULT = 50000; + // -------------------------------------------------------------------------- + // ENDPOINT + // -------------------------------------------------------------------------- + String ENDPOINT_KEY = "endpoint"; + String ENDPOINT_DEFAULT = "oss-cn-hangzhou.aliyuncs.com";// default is 1 + + // -------------------------------------------------------------------------- + // AUTHENTICATION + // -------------------------------------------------------------------------- + + String AUTH_USERNAME_KEY = "accesskey"; + String AUTH_USERNAME_DEFAULT = ""; + + String AUTH_PASSWORD_KEY = "secretkey"; + String AUTH_PASSWORD_DEFAULT = ""; + + // -------------------------------------------------------------------------- + // CLIENT CONFIGURATION + // -------------------------------------------------------------------------- + String PROXY_HOST_KEY = "proxyhost"; + String PROXY_PORT_KEY = "proxyport"; + + // -------------------------------------------------------------------------- + // PATH STYLE ACCESS + // -------------------------------------------------------------------------- + String PATH_STYLE_ACCESS_KEY = "path_style_access"; + boolean PATH_STYLE_ACCESS_DEFAULT = false; + + // -------------------------------------------------------------------------- + // CONTEXT NEEDS FROM AUTH MODULE + // -------------------------------------------------------------------------- + String OSSCLIENT_KEY = "ossclient"; + +} diff --git a/dev/cosbench-s3/.classpath b/dev/cosbench-s3/.classpath index c836559b..d5ba8291 100644 --- a/dev/cosbench-s3/.classpath +++ b/dev/cosbench-s3/.classpath @@ -3,21 +3,26 @@ - - - - - - - + + + + + + + + + + + + diff --git a/dev/cosbench-s3/META-INF/MANIFEST.MF b/dev/cosbench-s3/META-INF/MANIFEST.MF index 5aa9c0ad..e81b0826 100644 --- a/dev/cosbench-s3/META-INF/MANIFEST.MF +++ b/dev/cosbench-s3/META-INF/MANIFEST.MF @@ -6,22 +6,27 @@ Bundle-Version: 0.4.1.0 Bundle-Vendor: intel Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Bundle-ClassPath: ., - com.amazonaws-1.0.0.jar, + aws-java-sdk-s3-1.10.76.jar, + aws-java-sdk-kms-1.10.76.jar, + aws-java-sdk-core-1.10.76.jar, aspectjrt.jar, aspectjweaver.jar, - commons-codec-1.3.jar, - commons-logging-1.1.1.jar, + commons-codec-1.6.jar, + commons-logging-1.1.3.jar, freemarker-2.3.20.jar, - httpclient-4.1.1.jar, - httpcore-4.1.jar, + httpclient-4.3.6.jar, + httpcore-4.3.3.jar, jackson-core-asl-1.8.7.jar, jackson-mapper-asl-1.8.7.jar, + jackson-databind-2.5.3.jar, + jackson-annotations-2.5.0.jar, + jackson-core-2.5.3.jar, + jackson-dataformat-cbor-2.5.3.jar, + joda-time-2.8.1.jar, mail-1.4.3.jar, spring-beans-3.0.7.jar, spring-context-3.0.7.jar, - spring-core-3.0.7.jar, - stax-1.2.0.jar, - stax-api-1.0.1.jar + spring-core-3.0.7.jar Import-Package: com.intel.cosbench.api.auth, com.intel.cosbench.api.context, com.intel.cosbench.api.storage, diff --git a/dev/cosbench-s3/aws-java-sdk-core-1.10.76.jar b/dev/cosbench-s3/aws-java-sdk-core-1.10.76.jar new file mode 100644 index 00000000..6b24595a Binary files /dev/null and b/dev/cosbench-s3/aws-java-sdk-core-1.10.76.jar differ diff --git a/dev/cosbench-s3/aws-java-sdk-kms-1.10.76.jar b/dev/cosbench-s3/aws-java-sdk-kms-1.10.76.jar new file mode 100644 index 00000000..d3551b5a Binary files /dev/null and b/dev/cosbench-s3/aws-java-sdk-kms-1.10.76.jar differ diff --git a/dev/cosbench-s3/aws-java-sdk-s3-1.10.76.jar b/dev/cosbench-s3/aws-java-sdk-s3-1.10.76.jar new file mode 100644 index 00000000..2d19a17b Binary files /dev/null and b/dev/cosbench-s3/aws-java-sdk-s3-1.10.76.jar differ diff --git a/dev/cosbench-s3/build.properties b/dev/cosbench-s3/build.properties index d1e947ee..eb7bb7b3 100644 --- a/dev/cosbench-s3/build.properties +++ b/dev/cosbench-s3/build.properties @@ -1,36 +1,25 @@ source.. = src/ output.. = bin/ bin.includes = META-INF/,\ - com.amazonaws-1.0.0.jar,\ .,\ + aws-java-sdk-s3-1.10.76.jar,\ + aws-java-sdk-kms-1.10.76.jar,\ + aws-java-sdk-core-1.10.76.jar,\ aspectjrt.jar,\ aspectjweaver.jar,\ - commons-codec-1.3.jar,\ - commons-logging-1.1.1.jar,\ + commons-codec-1.6.jar,\ + commons-logging-1.1.3.jar,\ freemarker-2.3.20.jar,\ - httpclient-4.1.1.jar,\ - httpcore-4.1.jar,\ + httpclient-4.3.6.jar,\ + httpcore-4.3.3.jar,\ jackson-core-asl-1.8.7.jar,\ jackson-mapper-asl-1.8.7.jar,\ + jackson-databind-2.5.3.jar,\ + jackson-annotations-2.5.0.jar,\ + jackson-core-2.5.3.jar,\ + jackson-dataformat-cbor-2.5.3.jar,\ + joda-time-2.8.1.jar,\ mail-1.4.3.jar,\ spring-beans-3.0.7.jar,\ spring-context-3.0.7.jar,\ - spring-core-3.0.7.jar,\ - stax-1.2.0.jar,\ - stax-api-1.0.1.jar - .,\ - aspectjrt.jar,\ - aspectjweaver.jar,\ - commons-codec-1.3.jar,\ - commons-logging-1.1.1.jar,\ - freemarker-2.3.20.jar,\ - httpclient-4.1.1.jar,\ - httpcore-4.1.jar,\ - jackson-core-asl-1.8.7.jar,\ - jackson-mapper-asl-1.8.7.jar,\ - mail-1.4.3.jar,\ - spring-beans-3.0.7.jar,\ - spring-context-3.0.7.jar,\ - spring-core-3.0.7.jar,\ - stax-1.2.0.jar,\ - stax-api-1.0.1.jar + spring-core-3.0.7.jar diff --git a/dev/cosbench-s3/com.amazonaws-1.0.0.jar b/dev/cosbench-s3/com.amazonaws-1.0.0.jar deleted file mode 100644 index 8a4de1aa..00000000 Binary files a/dev/cosbench-s3/com.amazonaws-1.0.0.jar and /dev/null differ diff --git a/dev/cosbench-s3/commons-codec-1.6.jar b/dev/cosbench-s3/commons-codec-1.6.jar new file mode 100644 index 00000000..ee1bc49a Binary files /dev/null and b/dev/cosbench-s3/commons-codec-1.6.jar differ diff --git a/dev/cosbench-s3/commons-logging-1.1.3.jar b/dev/cosbench-s3/commons-logging-1.1.3.jar new file mode 100644 index 00000000..ab512540 Binary files /dev/null and b/dev/cosbench-s3/commons-logging-1.1.3.jar differ diff --git a/dev/cosbench-s3/httpclient-4.1.1.jar b/dev/cosbench-s3/httpclient-4.1.1.jar deleted file mode 100644 index c845ef96..00000000 Binary files a/dev/cosbench-s3/httpclient-4.1.1.jar and /dev/null differ diff --git a/dev/cosbench-s3/httpclient-4.3.6.jar b/dev/cosbench-s3/httpclient-4.3.6.jar new file mode 100755 index 00000000..091498c9 Binary files /dev/null and b/dev/cosbench-s3/httpclient-4.3.6.jar differ diff --git a/dev/cosbench-s3/httpcore-4.1.jar b/dev/cosbench-s3/httpcore-4.1.jar deleted file mode 100644 index a357c076..00000000 Binary files a/dev/cosbench-s3/httpcore-4.1.jar and /dev/null differ diff --git a/dev/cosbench-s3/httpcore-4.3.3.jar b/dev/cosbench-s3/httpcore-4.3.3.jar new file mode 100755 index 00000000..a8747b0c Binary files /dev/null and b/dev/cosbench-s3/httpcore-4.3.3.jar differ diff --git a/dev/cosbench-s3/jackson-annotations-2.5.0.jar b/dev/cosbench-s3/jackson-annotations-2.5.0.jar new file mode 100755 index 00000000..62521310 Binary files /dev/null and b/dev/cosbench-s3/jackson-annotations-2.5.0.jar differ diff --git a/dev/cosbench-s3/jackson-core-2.5.3.jar b/dev/cosbench-s3/jackson-core-2.5.3.jar new file mode 100755 index 00000000..f3ae1ae5 Binary files /dev/null and b/dev/cosbench-s3/jackson-core-2.5.3.jar differ diff --git a/dev/cosbench-s3/jackson-databind-2.5.3.jar b/dev/cosbench-s3/jackson-databind-2.5.3.jar new file mode 100755 index 00000000..e2171874 Binary files /dev/null and b/dev/cosbench-s3/jackson-databind-2.5.3.jar differ diff --git a/dev/cosbench-s3/jackson-dataformat-cbor-2.5.3.jar b/dev/cosbench-s3/jackson-dataformat-cbor-2.5.3.jar new file mode 100755 index 00000000..e8ecc04c Binary files /dev/null and b/dev/cosbench-s3/jackson-dataformat-cbor-2.5.3.jar differ diff --git a/dev/cosbench-s3/joda-time-2.8.1.jar b/dev/cosbench-s3/joda-time-2.8.1.jar new file mode 100755 index 00000000..94be6599 Binary files /dev/null and b/dev/cosbench-s3/joda-time-2.8.1.jar differ diff --git a/dev/cosbench-s3/src/com/intel/cosbench/api/S3Stor/S3Storage.java b/dev/cosbench-s3/src/com/intel/cosbench/api/S3Stor/S3Storage.java index 09621244..509c56be 100644 --- a/dev/cosbench-s3/src/com/intel/cosbench/api/S3Stor/S3Storage.java +++ b/dev/cosbench-s3/src/com/intel/cosbench/api/S3Stor/S3Storage.java @@ -53,6 +53,9 @@ public void init(Config config, Logger logger) { ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setConnectionTimeout(timeout); + clientConf.setSocketTimeout(timeout); + clientConf.withUseExpectContinue(false); + clientConf.withSignerOverride("S3SignerType"); // clientConf.setProtocol(Protocol.HTTP); if((!proxyHost.equals(""))&&(!proxyPort.equals(""))){ clientConf.setProxyHost(proxyHost); diff --git a/dev/cosbench-s3/stax-1.2.0.jar b/dev/cosbench-s3/stax-1.2.0.jar deleted file mode 100644 index 0df112a9..00000000 Binary files a/dev/cosbench-s3/stax-1.2.0.jar and /dev/null differ diff --git a/dev/cosbench-s3/stax-api-1.0.1.jar b/dev/cosbench-s3/stax-api-1.0.1.jar deleted file mode 100644 index d9a16651..00000000 Binary files a/dev/cosbench-s3/stax-api-1.0.1.jar and /dev/null differ diff --git a/release/cli.sh b/release/cli.sh old mode 100644 new mode 100755 diff --git a/release/conf/.driver/config.ini b/release/conf/.driver/config.ini index 768259e9..774a722e 100644 --- a/release/conf/.driver/config.ini +++ b/release/conf/.driver/config.ini @@ -67,11 +67,15 @@ plugins/cosbench-api@7\:start,\ plugins/cosbench-mock@7\:start,\ plugins/cosbench-ampli@7\:start,\ plugins/cosbench-swift@7\:start,\ +plugins/cosbench-openio@7\:start,\ plugins/cosbench-keystone@7\:start,\ plugins/cosbench-scality@7\:start,\ plugins/cosbench-swauth@7\:start,\ plugins/cosbench-httpauth@7:\start,\ plugins/cosbench-s3@7\:start,\ +plugins/cosbench-oss@7\:start,\ +plugins/cosbench-ecs@7\:start,\ +plugins/cosbench-gcs@7\:start,\ plugins/cosbench-librados@7\:start,\ plugins/cosbench-cdmi-base@7\:start,\ plugins/cosbench-cdmi-swift@7\:start,\ diff --git a/release/conf/aliyun-config-sample.xml b/release/conf/aliyun-config-sample.xml new file mode 100644 index 00000000..03c7ffbd --- /dev/null +++ b/release/conf/aliyun-config-sample.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/release/conf/ecs-config-sample.xml b/release/conf/ecs-config-sample.xml new file mode 100644 index 00000000..aa97c718 --- /dev/null +++ b/release/conf/ecs-config-sample.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/release/conf/gcs-config-sample.xml b/release/conf/gcs-config-sample.xml new file mode 100644 index 00000000..1a29288e --- /dev/null +++ b/release/conf/gcs-config-sample.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/release/conf/gcs-service-account-sample.json b/release/conf/gcs-service-account-sample.json new file mode 100644 index 00000000..c589bd98 --- /dev/null +++ b/release/conf/gcs-service-account-sample.json @@ -0,0 +1,12 @@ +// Service account json file can be generated from https://console.developers.google.com/apis/credentials +{ + "type": "service_account", + "private_key_id": "****************************************", + "private_key": "-----BEGIN PRIVATE KEY-----\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n-----END PRIVATE KEY-----\n", + "client_email": "*****@*****.****", + "client_id": "************************", + "auth_uri": "https://****.****.***/", + "token_uri": "https://****.****.***/", + "auth_provider_x509_cert_url": "https://****.****.***/", + "client_x509_cert_url": "https://****.****.***/" +} diff --git a/release/conf/openio-config-sample.xml b/release/conf/openio-config-sample.xml new file mode 100644 index 00000000..cbbcb668 --- /dev/null +++ b/release/conf/openio-config-sample.xml @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/release/cosbench-start.sh b/release/cosbench-start.sh old mode 100644 new mode 100755 diff --git a/release/cosbench-stop.sh b/release/cosbench-stop.sh old mode 100644 new mode 100755 diff --git a/release/start-all.sh b/release/start-all.sh old mode 100644 new mode 100755 diff --git a/release/start-controller.sh b/release/start-controller.sh old mode 100644 new mode 100755 diff --git a/release/start-driver.sh b/release/start-driver.sh old mode 100644 new mode 100755 index 33d3a71d..d1644a03 --- a/release/start-driver.sh +++ b/release/start-driver.sh @@ -22,7 +22,7 @@ SERVICE_NAME=driver VERSION=`cat VERSION` -OSGI_BUNDLES="cosbench-log_${VERSION} cosbench-tomcat_${VERSION} cosbench-config_${VERSION} cosbench-http_${VERSION} cosbench-cdmi-util_${VERSION} cosbench-core_${VERSION} cosbench-core-web_${VERSION} cosbench-api_${VERSION} cosbench-mock_${VERSION} cosbench-ampli_${VERSION} cosbench-swift_${VERSION} cosbench-keystone_${VERSION} cosbench-httpauth_${VERSION} cosbench-s3_${VERSION} cosbench-librados_${VERSION} cosbench-scality_${VERSION} cosbench-cdmi-swift_${VERSION} cosbench-cdmi-base_${VERSION} cosbench-driver_${VERSION} cosbench-driver-web_${VERSION}" +OSGI_BUNDLES="cosbench-log_${VERSION} cosbench-tomcat_${VERSION} cosbench-config_${VERSION} cosbench-http_${VERSION} cosbench-cdmi-util_${VERSION} cosbench-core_${VERSION} cosbench-core-web_${VERSION} cosbench-api_${VERSION} cosbench-mock_${VERSION} cosbench-ampli_${VERSION} cosbench-openio_${VERSION} cosbench-swift_${VERSION} cosbench-keystone_${VERSION} cosbench-httpauth_${VERSION} cosbench-s3_${VERSION} cosbench-oss_${VERSION} cosbench-gcs_${VERSION} cosbench-librados_${VERSION} cosbench-scality_${VERSION} cosbench-ecs_${VERSION} cosbench-cdmi-swift_${VERSION} cosbench-cdmi-base_${VERSION} cosbench-driver_${VERSION} cosbench-driver-web_${VERSION}" OSGI_CONSOLE_PORT=18089 diff --git a/release/stop-all.sh b/release/stop-all.sh old mode 100644 new mode 100755 diff --git a/release/stop-controller.sh b/release/stop-controller.sh old mode 100644 new mode 100755 diff --git a/release/stop-driver.sh b/release/stop-driver.sh old mode 100644 new mode 100755