Skip to content

Commit c78574c

Browse files
committed
bloblib v0.0.4
1 parent a8c5f3f commit c78574c

File tree

6 files changed

+122
-12
lines changed

6 files changed

+122
-12
lines changed

README.md

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,13 @@ BlobLib is the core libraries of the blobfs and blobfs-win, blobfs is a distribu
1111
Packaged the core operations of the azure blob storage as a separated library, this will make it easier for supporting different OS platform. This library can also be used alone with various projects.
1212

1313

14-
## Features:
15-
* New Features since v0.0.4 :
14+
## Features and Updates:
15+
### New features since v0.0.4 :
16+
17+
* upload and down with multiple threads, more faster and stable.
18+
* Page blob is full supported.
19+
20+
### New features since v0.0.3 :
1621

1722
* Support man operations of the container,virtual directory and blob. such as create, delete, list, read, write etc.
1823
* New extension class of the InputStream and OutputStream. add a new cache layer and is optimized for the blob read and write.

src/main/java/com/wesley/bloblib/BlobBufferedOus.java

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public class BlobBufferedOus extends OutputStream {
3636
int chunkSizeOfBB = Constants.BLOB_BUFFERED_OUTS_BLOCKBLOB_CHUNK_SIZE * 2;
3737
int chunkNumber = 0;
3838
int chunkSizeOfAB = Constants.BLOB_BUFFERED_OUTS_APPENDBLOB_CHUNK_SIZE;
39+
int chunkSizeOfPB = Constants.BLOB_BUFFERED_OUTS_PAGEBLOB_CHUNK_SIZE;
3940
/* the available bytes in central buffer */
4041
long blobOffset = 0;
4142
/* the path of the blob */
@@ -98,6 +99,7 @@ public synchronized void write(final byte[] data, final int offset, final int le
9899
/* write data to the buffer */
99100
writeToBuffer(data, offset, length);
100101
/* check the buffered data and the chunk size threshold */
102+
/* make sure the data is 512 Bytes aligned for page blob */
101103
if (isBufferedDataReadyToUpload()){
102104
int numOfdataUploaded = 0;
103105
if ((numOfdataUploaded = uploadBlobChunk(centralBuffer, 0, centralBufOffset)) > 0){
@@ -121,6 +123,12 @@ public synchronized final void flush() throws IOException {
121123
try {
122124
if (centralBufOffset > 0) {
123125
int numOfdataUploaded = 0;
126+
/* for page blob, we should make sure the length is 512 Bytes aligned */
127+
if (BfsBlobType.PAGEBLOB.equals(this.blobType)) {
128+
/* make sure the data is 512 Bytes aligned for page blob */
129+
this.centralBufOffset = (int) BfsUtility.extendTo512BytesAlignedLength(centralBufOffset);
130+
this.centralBuffer = BfsUtility.extendTo512BytesAlignedData(this.centralBuffer);
131+
}
124132
if ((numOfdataUploaded = uploadBlobChunk(centralBuffer, 0, centralBufOffset)) > 0) {
125133
totalDataUploaded += numOfdataUploaded;
126134
/* reset the chunk count */
@@ -198,17 +206,20 @@ public synchronized final int uploadBlobChunk (byte[] rawData, int offset, int
198206
/* renew the lease firstly, otherwise the lease may be expired, this will cause error */
199207
blob.renewLease(accCondtion);
200208
if (BfsBlobType.BLOCKBLOB.equals(this.blobType)){
209+
/* is the block blob */
201210
int uploadRes[] = parallelUploader.uploadBlobWithParallelThreads(rawData, blob, offset, length,
202211
blockList, accCondtion, leaseID, chunkNumber);
203212
/* update the chunk counter */
204213
chunkNumber = uploadRes[1];
205214
}
206215
else
207216
{
208-
ByteArrayInputStream bInput = new ByteArrayInputStream(rawData, offset, length);
217+
/* is the append blob or page blob */
218+
ByteArrayInputStream bInput = new ByteArrayInputStream(rawData, offset, length);
209219
if (BfsBlobType.APPENDBLOB.equals(this.blobType)){
210220
((CloudAppendBlob) blob).appendBlock(bInput, (long)length, accCondtion, null, null);
211221
}else if (BfsBlobType.PAGEBLOB.equals(this.blobType)){
222+
/* for page blob, we already make sure the length is 512B aligned */
212223
((CloudPageBlob) blob).upload(bInput, length, accCondtion, null, null);
213224
}
214225
bInput.close();
@@ -223,12 +234,19 @@ public synchronized final int uploadBlobChunk (byte[] rawData, int offset, int
223234
return dataUploadedThisChunk;
224235
}
225236

237+
/** make sure the length is valid
238+
* @return
239+
*/
226240
public synchronized boolean isBufferedDataReadyToUpload() {
227241
boolean result = false;
228242
if (BfsBlobType.BLOCKBLOB.equals(this.blobType)){
229-
if (centralBufOffset > chunkSizeOfBB){ return result = true;}
243+
if (centralBufOffset > chunkSizeOfBB){ result = true;}
230244
} else if (BfsBlobType.APPENDBLOB.equals(this.blobType)){
231-
if (centralBufOffset > chunkSizeOfAB){ return result = true;}
245+
if (centralBufOffset > chunkSizeOfAB){ result = true;}
246+
} else if(BfsBlobType.PAGEBLOB.equals(this.blobType)){
247+
if (centralBufOffset > chunkSizeOfPB && BfsUtility.is512BytesAligned(centralBufOffset)){
248+
result = true;
249+
}
232250
}
233251
/* page blob */
234252
return result;
@@ -259,14 +277,16 @@ public synchronized void verifyUploadConditions() throws BfsException {
259277
String errMessage = "The size of the source file exceeds the size limit: " + blobSizeLimit + ".";
260278
throw new BfsException(errMessage);
261279
}
262-
263-
if (numOfCommitedBlocks + chunkNumber > Constants.BLOB_BLOCK_NUMBER_LIMIT - 1){ //50000
264-
String errMessage = "The block count of target blob: " + fullBlobPath + " exceeds the " + Constants.BLOB_BLOCK_NUMBER_LIMIT + " count limit.";
265-
throw new BfsException(errMessage);
280+
/* for block blob and append blob */
281+
if (BfsBlobType.BLOCKBLOB.equals(this.blobType) || (BfsBlobType.APPENDBLOB.equals(this.blobType))){
282+
if (numOfCommitedBlocks + chunkNumber > Constants.BLOB_BLOCK_NUMBER_LIMIT - 1){ //50000
283+
String errMessage = "The block count of target blob: " + fullBlobPath + " exceeds the " + Constants.BLOB_BLOCK_NUMBER_LIMIT + " count limit.";
284+
throw new BfsException(errMessage);
285+
}
266286
}
267287
} catch (Exception ex) {
268288
String errMessage = "Unexpected exception occurred when verifing the upload conditons for to the blob : "
269-
+ this.fullBlobPath + "." + ex.getMessage();
289+
+ this.fullBlobPath + ". " + ex.getMessage();
270290
BfsUtility.throwBlobfsException(ex, errMessage);
271291

272292
}

src/main/java/com/wesley/bloblib/BlobLib.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ public class BlobLib {
77
@SuppressWarnings("static-access")
88
public static void main(String[] args) {
99
try {
10+
// long len = get512BytesAlignedLength(23435543L);
11+
// System.out.println(len);
1012
//BfsPath bfsPath = new BfsPath("/index.html");
1113
//PathProperties pathProperties = bfsPath.getBfsPathProperties();
1214
// MsgsToBeSentCache.getInstance().startAutoSendService();
@@ -36,9 +38,9 @@ public static void main(String[] args) {
3638
// BlobBufferedIns bbIns = new BlobBufferedIns(insParams);
3739
// BlobReqParams ousParams = new BlobReqParams();
3840
// ousParams.setContainer("music2017");
39-
// ousParams.setBlob("blobfs-2017-11-23.log");
41+
// ousParams.setBlob("blobfs-2017-10-21.vhd");
4042
//// /* get the blob type */
41-
// ousParams.setBfsBlobType(BfsBlobType.BLOCKBLOB);
43+
// ousParams.setBfsBlobType(BfsBlobType.PAGEBLOB);
4244
// BlobService.createBlob(ousParams);
4345
// BlobBufferedOus bbOus = new BlobBufferedOus(ousParams);
4446
// FileReader fileRead = new FileReader("C:\\Downloads\\Mydevs\\blobfs-win\\blobfs-win\\bin\\Debug\\blobfs-2017-11-19.log");

src/main/java/com/wesley/bloblib/Constants.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ public final class Constants {
2020
public static final int BLOB_BUFFERED_OUTS_BUFFER_SIZE = 4 * 1024 * 1024; //default is 4MB: 4 * 1024 * 1024 = 4194304
2121
public static final int BLOB_BUFFERED_OUTS_BLOCKBLOB_CHUNK_SIZE = 4 * 1024 * 1024; //default is 4MB: 4 * 1024 * 1024 = 4194304
2222
public static final int BLOB_BUFFERED_OUTS_APPENDBLOB_CHUNK_SIZE = 4 * 1024 * 1024; //default is 4MB: 4 * 1024 * 1024 = 4194304
23+
public static final int BLOB_BUFFERED_OUTS_PAGEBLOB_CHUNK_SIZE = 4 * 1024 * 1024; //default is 4MB: 4 * 1024 * 1024 = 4194304
2324

2425
public static final int BFS_FILES_CACHE_INIT_CAPACITY = 1024; //1024
2526
public static final int BFS_FILES_CACHE_MAX_CAPACITY = 65535; //opened file

src/main/java/com/wesley/bloblib/ParallelUploader.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ private final List<UploadTask> splitJobIntoTheParallelTasks(int blobOffset, long
121121
} else {
122122
bytesToWrite = (int)bytesLeft;
123123
}
124+
124125
chunkNumber ++;
125126
/* save chunk id in array (must be base64) */
126127
String chunkId = DatatypeConverter.printBase64Binary(String.format("BlockId%07d", chunkNumber).getBytes(StandardCharsets.UTF_8));

src/main/java/com/wesley/bloblib/utils/BfsUtility.java

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import org.pmw.tinylog.Logger;
1717

18+
import com.fasterxml.jackson.core.json.ByteSourceJsonBootstrapper;
1819
import com.microsoft.azure.storage.blob.BlockEntry;
1920
import com.wesley.bloblib.Constants;
2021
import com.wesley.bloblib.BfsException;
@@ -109,6 +110,10 @@ public static int commandExecutor(final String command, final StringBuilder outp
109110
return process.exitValue();
110111
}
111112

113+
/**
114+
* @param process
115+
* @return
116+
*/
112117
public static boolean isRunning(Process process) {
113118
try {
114119
process.exitValue();
@@ -133,6 +138,10 @@ public static int getIdOnUnix(String cmd) throws IOException{
133138
return uid;
134139
}
135140

141+
/**
142+
* @param path
143+
* @return
144+
*/
136145
public static String removeLastSlash (String path){
137146
String tmpPath = "";
138147
if (path.endsWith("/")) {
@@ -143,24 +152,41 @@ public static String removeLastSlash (String path){
143152
return tmpPath;
144153
}
145154

155+
/**
156+
* @param date
157+
* @param timeZone
158+
* @return
159+
*/
146160
public static String changeDateTimeZone(Date date , String timeZone){
147161
DateFormat dateFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy", Locale.US);
148162
dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone));
149163
return dateFormat.format(date);
150164
}
151165

166+
/**
167+
* @param fileOrDirPath
168+
* @return
169+
*/
152170
public static String getParentDirPath(String fileOrDirPath) {
153171
boolean endsWithSlash = fileOrDirPath.endsWith(File.separator);
154172
return fileOrDirPath.substring(0, fileOrDirPath.lastIndexOf(File.separatorChar,
155173
endsWithSlash ? fileOrDirPath.length() - 2 : fileOrDirPath.length() - 1));
156174
}
157175

158176

177+
/**
178+
* @param date
179+
* @return
180+
*/
159181
public static String convertDateToString(Date date){
160182
DateFormat dateFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy", Locale.US);
161183
return dateFormat.format(date);
162184
}
163185

186+
/**
187+
* @param dateString
188+
* @return
189+
*/
164190
public static Date convertStringToDate(String dateString){
165191
DateFormat dateFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy", Locale.US);
166192
Date date = null;
@@ -173,8 +199,63 @@ public static Date convertStringToDate(String dateString){
173199
return date;
174200
}
175201

202+
/**
203+
* @return
204+
*/
176205
public static boolean isWindows() {
177206
return (OS.indexOf("win") >= 0);
178207
}
179208

209+
/**
210+
* @param length
211+
* @return
212+
*/
213+
public static boolean is512BytesAligned(long length){
214+
if (length % 512 == 0){return true;}
215+
return false;
216+
}
217+
218+
/**
219+
* get the 512 bytes aligned length
220+
* @param length
221+
* @return
222+
*/
223+
public static long shrinkTo512BytesAlignedLength(long length){
224+
long tmpLen = 0;
225+
if (length % 512 == 0){tmpLen = length; }
226+
else{
227+
if (length < 512){tmpLen = 0; }
228+
else{
229+
tmpLen = (length - length % 512);
230+
}
231+
}
232+
return tmpLen;
233+
}
234+
235+
public static long extendTo512BytesAlignedLength(long length){
236+
long tmpLen = 0;
237+
if (length % 512 == 0){ tmpLen = length; }
238+
else{
239+
tmpLen = length + (512 - length % 512);
240+
}
241+
return tmpLen;
242+
}
243+
244+
/**
245+
* make data 512 bytes aligned by padding the rest slots with zero;
246+
* @param origData
247+
* @return
248+
*/
249+
public static byte[] extendTo512BytesAlignedData(byte[] origData){
250+
long length = origData.length;
251+
if (length % 512 == 0){ return origData; }
252+
else{
253+
long tmpLen = length + (512 - length % 512);
254+
byte[] tgtData = new byte [(int) tmpLen]; // all bytes are initialized with 0's
255+
System.arraycopy(origData, 0, tgtData, 0, (int) length);
256+
return tgtData;
257+
}
258+
}
259+
260+
180261
}

0 commit comments

Comments
 (0)