Skip to content

Compression size test #321

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Jonathan25021 wants to merge 5 commits into
base: CompressionSizeTest
Choose a base branch
from
Open

Compression size test #321

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
65865f6
Added Excel to CSV file converter
Jonathan25021 Apr 6, 2021
5ce273e
merged origin to upstream
Jonathan25021 Jun 2, 2021
94c1b03
Changed TYPEFIVE to include gaps 1-5
Jonathan25021 Nov 9, 2021
15eb645
Merge branch 'master' of https://github.com/Jonathan25021/dataspread-…
Jonathan25021 Nov 9, 2021
c2cfe43
Added TYPESIX (FF GapN); still a little buggy
Jonathan25021 Nov 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions testcode/src/CompressionSizeTest/SizeTestMain.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@ public class SizeTestMain {
/**
* DB Connection Configuration
* */
public static String url = "jdbc:postgresql://127.0.0.1:5432/dbtest";
public static String url = "jdbc:postgresql://127.0.0.1:5432/SizeTest";
public static String dbDriver = "org.postgresql.Driver";
public static String userName = "totemtang";
public static String password = "1234";
public static String userName = "dataspreaduser";
public static String password = "password";


public static void main (String[] args) {
Expand Down Expand Up @@ -95,8 +95,9 @@ public static void main (String[] args) {
int patternIdx = precMeta.getEdgeMeta().patternType.ordinal();
compEdges[patternIdx] = compEdges[patternIdx] + 1;

int compressCount =
precMeta.getPatternType() == PatternType.TYPEFIVE ? (dep.getCellCount() + 1)/2 : dep.getCellCount();
int compressCount = (dep.getCellCount() + precMeta.getEdgeMeta().gapLength) /
(precMeta.getEdgeMeta().gapLength + 1);

uncompEdges[patternIdx] = uncompEdges[patternIdx] + compressCount;
});
});
Expand Down
123 changes: 57 additions & 66 deletions zssmodel/src/org/zkoss/zss/model/impl/sys/DependencyTableComp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ public class DependencyTableComp extends DependencyTableAdv {

private CompressInfoComparator compressInfoComparator = new CompressInfoComparator();

private final int MAX_GAP = 4;

public DependencyTableComp() {}

public Map<Ref, List<RefWithMeta>> getAllEdges() {
Expand Down Expand Up @@ -162,7 +164,8 @@ private void flushUpdateCache(LinkedList<EdgeUpdate> updateCache,
try {
deleteDBEntry(dbContext, edgeUpdate.oldPrec, edgeUpdate.oldDep, edgeUpdate.oldEdgeMeta);
insertDBEntry(dbContext, edgeUpdate.newPrec, edgeUpdate.newDep, edgeUpdate.newEdgeMeta.patternType,
edgeUpdate.newEdgeMeta.startOffset, edgeUpdate.newEdgeMeta.endOffset);
edgeUpdate.newEdgeMeta.startOffset, edgeUpdate.newEdgeMeta.endOffset,
edgeUpdate.newEdgeMeta.gapLength);
} catch (SQLException e) {
e.printStackTrace();
}
Expand All @@ -185,7 +188,7 @@ private void fastModify(Ref prec, Ref dep, boolean isInsert,
Ref newDep = selectedInfo.dep.getBoundingBox(selectedInfo.candDep);
Pair<Offset, Offset> offsetPair = computeOffset(newPrec, newDep, selectedInfo.compType);
updateMatch.updateEdge(newPrec, newDep,
new EdgeMeta(selectedInfo.compType, offsetPair.getX(), offsetPair.getY()));
new EdgeMeta(selectedInfo.compType, updatePair.getX().edgeMeta.gapLength, offsetPair.getX(), offsetPair.getY()));
updateCache.addFirst(updateMatch);
}
} else {
Expand All @@ -199,11 +202,12 @@ private void fastModify(Ref prec, Ref dep, boolean isInsert,
Pair<Offset, Offset> offsetPair = computeOffset(newPrec, newDep, selectedInfo.compType);

EdgeUpdate evicted = addToUpdateCache(updateCache, newPrec, newDep,
new EdgeMeta(selectedInfo.compType, offsetPair.getX(), offsetPair.getY()));
new EdgeMeta(selectedInfo.compType, selectedInfo.edgeMeta.gapLength, offsetPair.getX(), offsetPair.getY()));
if (evicted != null) {
deleteDBEntry(dbContext, evicted.oldPrec, evicted.oldDep, evicted.oldEdgeMeta);
insertDBEntry(dbContext, evicted.newPrec, evicted.newDep, evicted.newEdgeMeta.patternType,
evicted.newEdgeMeta.startOffset, evicted.newEdgeMeta.endOffset);
evicted.newEdgeMeta.startOffset, evicted.newEdgeMeta.endOffset,
evicted.newEdgeMeta.gapLength);
}
} else {
EdgeMeta noTypeEdgeMeta = new EdgeMeta(PatternType.NOTYPE,
Expand All @@ -212,7 +216,8 @@ private void fastModify(Ref prec, Ref dep, boolean isInsert,
if (evicted != null) {
deleteDBEntry(dbContext, evicted.oldPrec, evicted.oldDep, evicted.oldEdgeMeta);
insertDBEntry(dbContext, evicted.newPrec, evicted.newDep, evicted.newEdgeMeta.patternType,
evicted.newEdgeMeta.startOffset, evicted.newEdgeMeta.endOffset);
evicted.newEdgeMeta.startOffset, evicted.newEdgeMeta.endOffset,
evicted.newEdgeMeta.gapLength);
}
}
} catch (SQLException e) {
Expand Down Expand Up @@ -269,7 +274,7 @@ public void refreshCache(String bookName, String sheetName) {
private void loadEverything(String bookName, String sheetName) {
_rectToRefCache = RTree.create();
String selectQuery =
" SELECT range::box, dep_range::box, pattern_type, offsetRange::box" +
" SELECT range::box, dep_range::box, pattern_type, offsetRange::box, gap_length" +
" FROM " + dependencyTableName +
" WHERE bookname = ?" +
" AND sheetname = ?";
Expand All @@ -292,8 +297,9 @@ private void loadEverything(String bookName, String sheetName) {
(int) offsetRange.point[0].y);
Offset endOffset = new Offset((int) offsetRange.point[1].x,
(int) offsetRange.point[1].y);
int gapLength = rs.getInt(5);
EdgeMeta edgeMeta =
new EdgeMeta(patternType, startOffset, endOffset);
new EdgeMeta(patternType, gapLength, startOffset, endOffset);
insertMemEntry(prec, dep, edgeMeta);
}
} catch (SQLException e) {
Expand Down Expand Up @@ -415,7 +421,7 @@ private void performOneInsert(Ref prec, Ref dep) {
DBContext dbContext = new DBContext(connection);
if (compressInfoList.isEmpty()) {
insertDBEntry(dbContext, prec, dep, PatternType.NOTYPE,
Offset.noOffset, Offset.noOffset);
Offset.noOffset, Offset.noOffset, 0);
} else {
CompressInfo selectedInfo =
Collections.min(compressInfoList, compressInfoComparator);
Expand All @@ -436,7 +442,7 @@ private void updateOneCompressEntry(DBContext dbContext,
Ref newDep = selectedInfo.dep.getBoundingBox(selectedInfo.candDep);
Pair<Offset, Offset> offsetPair = computeOffset(newPrec, newDep, selectedInfo.compType);
insertDBEntry(dbContext, newPrec, newDep, selectedInfo.compType,
offsetPair.x, offsetPair.y);
offsetPair.x, offsetPair.y, 0);
}

private Pair<CompressInfo, EdgeUpdate> findCompressInfoInUpdateCache(Ref prec, Ref dep,
Expand All @@ -455,8 +461,8 @@ private Pair<CompressInfo, EdgeUpdate> findCompressInfoInUpdateCache(Ref prec, R

if (updatePair == null) {
for (EdgeUpdate oneUpdate: updateCache) {
CompressInfo compRes = findCompressionPatternGapOne(prec, dep,
oneUpdate.newPrec, oneUpdate.newDep, oneUpdate.newEdgeMeta);
CompressInfo compRes = findCompressionPatternGapOneToN(prec, dep,
oneUpdate.newPrec, oneUpdate.newDep, oneUpdate.newEdgeMeta, MAX_GAP);
PatternType compType = compRes.compType;
if (compType != PatternType.NOTYPE) {
updatePair = new Pair<>(compRes, oneUpdate);
Expand Down Expand Up @@ -532,13 +538,15 @@ private void deleteDBEntry(DBContext dbContext,
" AND sheetname = ?" +
" AND range ~= ?" +
" AND dep_range ~= ?" +
" AND offsetRange ~= ?";
" AND offsetRange ~= ?" +
" AND gap_length = ?";
PreparedStatement retStmt = dbContext.getConnection().prepareStatement(query);
retStmt.setString(1, prec.getBookName());
retStmt.setString(2, prec.getSheetName());
retStmt.setObject(3, RefUtils.refToPGBox(prec), Types.OTHER);
retStmt.setObject(4, RefUtils.refToPGBox(dep), Types.OTHER);
retStmt.setObject(5, RefUtils.offsetToPGBox(edgeMeta.startOffset, edgeMeta.endOffset), Types.OTHER);
retStmt.setInt(6, edgeMeta.gapLength);
retStmt.execute();
}

Expand All @@ -551,12 +559,13 @@ private void insertDBEntry(DBContext dbContext,
Ref newDep,
PatternType patternType,
Offset startOffset,
Offset endOffset) throws SQLException {
insertMemEntry(newPrec, newDep, new EdgeMeta(patternType, startOffset, endOffset));
Offset endOffset,
int gapLength) throws SQLException {
insertMemEntry(newPrec, newDep, new EdgeMeta(patternType, gapLength, startOffset, endOffset));

if (!refreshCacheMode) {
String query = "INSERT INTO " + dependencyTableName +
" VALUES (?,?,?,?,?,?,?,?,?)";
" VALUES (?,?,?,?,?,?,?,?,?,?)";
PreparedStatement retStmt = dbContext.getConnection().prepareStatement(query);
retStmt.setString(1, newPrec.getBookName());
retStmt.setString(2, newPrec.getSheetName());
Expand All @@ -567,6 +576,7 @@ private void insertDBEntry(DBContext dbContext,
retStmt.setBoolean(7, true);
retStmt.setInt(8, patternType.ordinal());
retStmt.setObject(9, RefUtils.offsetToPGBox(startOffset, endOffset));
retStmt.setInt(10, gapLength);
retStmt.execute();
}

Expand Down Expand Up @@ -653,7 +663,7 @@ private void performOneDelete(Ref delDep) {
} else {
try {
insertDBEntry(dbContext, newPrec, newDep, newEdgeMeta.patternType,
newEdgeMeta.startOffset, newEdgeMeta.endOffset);
newEdgeMeta.startOffset, newEdgeMeta.endOffset, newEdgeMeta.gapLength);
} catch (SQLException e) {
e.printStackTrace();
}
Expand Down Expand Up @@ -772,59 +782,39 @@ private CompressInfo findCompressionPattern(Ref prec, Ref dep,
prec, dep, candPrec, candDep, metaData);
}

private CompressInfo findCompressionPatternGapOne(Ref prec, Ref dep,
Ref candPrec, Ref candDep, EdgeMeta metaData) {
if (dep.getColumn() == candDep.getColumn() && candDep.getLastRow() - dep.getRow() == -2) {
if (metaData.patternType == PatternType.NOTYPE) {
Offset offsetStartA = RefUtils.refToOffset(prec, dep, true);
Offset offsetStartB = RefUtils.refToOffset(candPrec, candDep, true);

Offset offsetEndA = RefUtils.refToOffset(prec, dep, false);
Offset offsetEndB = RefUtils.refToOffset(candPrec, candDep, false);

if (offsetStartA.equals(offsetStartB) &&
offsetEndA.equals(offsetEndB)) {
return new CompressInfo(false, Direction.TODOWN, PatternType.TYPEFIVE,
prec, dep, candPrec, candDep, metaData);
}
} else if (metaData.patternType == PatternType.TYPEFIVE) {
Offset offsetStartA = RefUtils.refToOffset(prec, dep, true);
Offset offsetEndA = RefUtils.refToOffset(prec, dep, false);

if (offsetStartA.equals(metaData.startOffset) &&
offsetEndA.equals(metaData.endOffset)) {
return new CompressInfo(false, Direction.TODOWN, PatternType.TYPEFIVE,
prec, dep, candPrec, candDep, metaData);
}
}
} else if (dep.getRow() == candDep.getRow() && candDep.getLastColumn() - dep.getColumn() == -2) {
if (metaData.patternType == PatternType.NOTYPE) {
Offset offsetStartA = RefUtils.refToOffset(prec, dep, true);
Offset offsetStartB = RefUtils.refToOffset(candPrec, candDep, true);

Offset offsetEndA = RefUtils.refToOffset(prec, dep, false);
Offset offsetEndB = RefUtils.refToOffset(candPrec, candDep, false);

if (offsetStartA.equals(offsetStartB) &&
offsetEndA.equals(offsetEndB)) {
return new CompressInfo(false, Direction.TORIGHT, PatternType.TYPEFIVE,
prec, dep, candPrec, candDep, metaData);
}
} else if (metaData.patternType == PatternType.TYPEFIVE) {
Offset offsetStartA = RefUtils.refToOffset(prec, dep, true);
Offset offsetEndA = RefUtils.refToOffset(prec, dep, false);

if (offsetStartA.equals(metaData.startOffset) &&
offsetEndA.equals(metaData.endOffset)) {
return new CompressInfo(false, Direction.TORIGHT, PatternType.TYPEFIVE,
prec, dep, candPrec, candDep, metaData);
}
private CompressInfo findCompressionPatternGapOneToN(Ref prec, Ref dep,
Ref candPrec, Ref candDep, EdgeMeta metaData, int N) {
Direction direction = Direction.NODIRECTION;
PatternType pattern = PatternType.NOTYPE;
// if (dep.getColumn() == candDep.getColumn() && dep.getRow() - candDep.getLastRow() == gapLength + 1) {
if (dep.getColumn() == candDep.getColumn() && dep.getRow() != candDep.getRow()) {
direction = Direction.TODOWN;
//} else if (dep.getRow() == candDep.getRow() && dep.getColumn() - candDep.getLastColumn() == gapLength + 1) {
} else if (dep.getColumn() != candDep.getColumn() && dep.getRow() == candDep.getRow()) {
direction = Direction.TORIGHT;
}
if (direction == Direction.NODIRECTION) {
return new CompressInfo(false, Direction.NODIRECTION, PatternType.NOTYPE,
prec, dep, candPrec, candDep, metaData);
}
for (int gapLength = 1; gapLength <= N; gapLength++) {
if (isCompressibleTypeFive(candPrec, candDep, prec, dep, direction, gapLength, metaData)) {
metaData.setGapLength(gapLength);
pattern = PatternType.TYPEFIVE;
break;
} else if (isCompressibleTypeSix(candPrec, candDep, prec, dep, direction, gapLength)) {
metaData.setGapLength(gapLength);
pattern = PatternType.TYPESIX;
break;
}

}
return new CompressInfo(false, Direction.NODIRECTION, PatternType.NOTYPE,

return new CompressInfo(false, direction, pattern,
prec, dep, candPrec, candDep, metaData);
}


private PatternType findCompPatternHelper(Direction direction,
Ref prec, Ref dep,
Ref candPrec, Ref candDep,
Expand Down Expand Up @@ -855,7 +845,7 @@ private Iterable<Ref> findOverlapAndAdjacency(Ref ref) {
findOverlappingRefs(ref).forEachRemaining(res::addLast);
Arrays.stream(Direction.values()).filter(direction -> direction != Direction.NODIRECTION)
.forEach(direction ->
findOverlappingRefs(shiftRef(ref, direction))
findOverlappingRefs(shiftRef(ref, direction, 1))
.forEachRemaining(adjRef -> {
if (isValidAdjacency(adjRef, ref)) res.addLast(adjRef); // valid adjacency
})
Expand Down Expand Up @@ -1099,7 +1089,8 @@ private class CompressInfo {
Direction direction,
PatternType compType,
Ref prec, Ref dep,
Ref candPrec, Ref candDep, EdgeMeta edgeMeta) {
Ref candPrec, Ref candDep,
EdgeMeta edgeMeta) {
this.isDuplicate = isDuplicate;
this.direction = direction;
this.compType = compType;
Expand Down
19 changes: 17 additions & 2 deletions zssmodel/src/org/zkoss/zss/model/impl/sys/utils/EdgeMeta.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,20 @@ public class EdgeMeta {
public final PatternType patternType;
public final Offset startOffset;
public final Offset endOffset;
public int gapLength;

public EdgeMeta(PatternType patternType,
Offset startOffset, Offset endOffset) {
this.patternType = patternType;
this.gapLength = 0;
this.startOffset = startOffset;
this.endOffset = endOffset;
}

public EdgeMeta(PatternType patternType, int gapLength,
Offset startOffset, Offset endOffset) {
this.patternType = patternType;
this.gapLength = gapLength;
this.startOffset = startOffset;
this.endOffset = endOffset;
}
Expand All @@ -21,8 +31,13 @@ public boolean equals(Object o) {
if (!(o instanceof EdgeMeta)) return false;
EdgeMeta edgeMeta = (EdgeMeta) o;
return patternType == edgeMeta.patternType &&
Objects.equals(startOffset, edgeMeta.startOffset) &&
Objects.equals(endOffset, edgeMeta.endOffset);
Objects.equals(this.startOffset, edgeMeta.startOffset) &&
Objects.equals(this.endOffset, edgeMeta.endOffset) &&
Objects.equals(this.gapLength, edgeMeta.gapLength);
}

public void setGapLength(int gapLength) {
this.gapLength = gapLength;
}

@Override
Expand Down
Loading