Skip to content

block_desc添加ts_range和ref_num2个字段 #461

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pengsven wants to merge 2 commits into
base: master
Choose a base branch
from
Open

block_desc添加ts_range和ref_num2个字段 #461

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
82612d9
add ts_range and ref_num for block_desc
Nov 20, 2019
4122a2c
add block_group.link_block
Nov 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions ectypes/block_desc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,16 +41,53 @@ def _mtime(ts=None):
return int(ts)


def _ts_range(ts_range=None):
if ts_range is None:
return ts_range

if ts_range[0] is not None:
ts_range[0] = str(ts_range[0])

if ts_range[1] is not None:
ts_range[1] = str(ts_range[1])

return ts_range


class BlockDesc(FixedKeysDict):

keys_default = dict(
block_id=_block_id,
size=int,
range=_range,
ts_range=_ts_range,
is_del=_is_del,
mtime=_mtime,
ref_num=int,
)

def mark_del(self):
if self['ref_num'] != 0:
raise ValueError("cannot mark del block with ref_num:{n} > 0".format(
n=self['ref_num']))

self["is_del"] = 1
self.mtime = int(time.time())

def is_mark_del(self):
return self['is_del'] == 1

def add_ref(self):
if self['is_del'] != 0:
raise ValueError("reference a block marked delete")

self['ref_num'] += 1

def rm_ref(self):
if self['ref_num'] < 1:
raise ValueError("ref_num:{n} < 1".format(n=self['ref_num']))

self['ref_num'] -= 1

def can_del(self):
return self['ref_num'] == 0
67 changes: 55 additions & 12 deletions ectypes/block_group.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,27 +100,55 @@ def make_type_map(self):

def mark_delete_block(self, block_index):
block = self.get_block(block_index, raise_error=True)
block.mark_del()

return block
block.rm_ref()

def delete_block(self, block_index):
block = self.get_block(block_index, raise_error=True)
del self['blocks'][str(block_index)]
if block.can_del():
block.mark_del()
return block

return block
return None

def mark_delete_block_byid(self, block_id):
block = self.get_block_byid(block_id, raise_error=True)
block.mark_del()

return block
block.rm_ref()

def delete_block_byid(self, block_id):
if block.can_del():
block.mark_del()
return block

return None

def unlink_block(self, block_index):
block = self.get_block(block_index, raise_error=True)

if not block.is_mark_del():
block.rm_ref()

if block.can_del():
del self['blocks'][str(block_index)]
return block

return None

def unlink_block_byid(self, block_id):
block = self.get_block_byid(block_id, raise_error=True)
del self['blocks'][block_id.block_index]

return block
if not block.is_mark_del():
block.rm_ref()

if block.can_del():
del self['blocks'][block_id.block_index]
return block

return None

def delete_block(self, block_index):
return self.unlink_block(block_index)

def delete_block_byid(self, block_id):
return self.unlink_block_byid(block_id)

def has(self, block):
bid = block['block_id']
Expand All @@ -129,8 +157,23 @@ def has(self, block):
existent = self['blocks'].get(bidx)
return existent == block

def link_block(self, block_index):
block = self.get_block(block_index, raise_error=True)

block.add_ref()
return block

def link_block_byid(self, block_id):
block = self.get_block_byid(block_id, raise_error=True)

block.add_ref()
return block

def add_block(self, new_block, replace=False, allow_exist=False):

if new_block['ref_num'] != 1:
raise BlockGroupBaseError("the ref_num of block that added must be 1")

if self.has(new_block) and allow_exist:
return new_block

Expand Down Expand Up @@ -251,7 +294,7 @@ def classify_blocks(self, idc_index, only_primary=True):
if blk is None:
continue

if blk['is_del'] == 1:
if blk.is_mark_del():
mark_del.append(blk)
continue

Expand Down
47 changes: 45 additions & 2 deletions ectypes/test/test_block_desc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,34 @@ def test_blockdesc(self):
{'block_id': None,
'size': 0,
'range': None,
'ts_range': None,
'ref_num': 0,
'is_del': 0}),

({'block_id': block_id,
'range': ['a', 'b'],
'ts_range': ["124", None],
'ref_num': 0,
'size': 34,
'mtime': 1,
'is_del': 0},
{'block_id': BlockID(block_id),
'range': rangeset.Range('a', 'b'),
'ts_range': ["124", None],
'ref_num': 0,
'size': 34,
'mtime': 1,
'is_del': 0}),

({'block_id': BlockID(block_id),
'range': rangeset.Range('b', 'bb'),
'ts_range': ["1235", "456"],
'ref_num': 0,
'mtime': 1},
{'block_id': BlockID(block_id),
'range': rangeset.Range('b', 'bb'),
'ts_range': ["1235", "456"],
'ref_num': 0,
'size': 0,
'mtime': 1,
'is_del': 0, })
Expand All @@ -61,15 +71,48 @@ def test_json(self):
DriveID('idc000' 'c62d8736c7280002'), 1),
'size': 1000,
'range': ['0a', '0b'],
'ts_range': ["1235", "456"],
'ref_num': 1,
'is_del': 0,
'mtime': 1,

})

rst = utfjson.dump(blk)
expected = ('{"is_del": 0, "range": ["0a", "0b"], "mtime": 1, "block_id": '
'"d0g0006400000001230000idc000c62d8736c72800020000000001", "size": 1000}')

expected = ('{"block_id": "d0g0006400000001230000idc000c62d8736c72800020000000001", "is_del": 0, "ref_num": 1, "range": ["0a", "0b"], "mtime": 1, "ts_range": ["1235", "456"], "size": 1000}')

self.assertEqual(expected, rst)
loaded = BlockDesc(utfjson.load(rst))
self.assertEqual(blk, loaded)

def test_ref(self):
block_id = 'd1g0006300000001230101idc000c62d8736c72800020000000001'
blk = BlockDesc({
'block_id': block_id,
'size': 1000,
'range': ['0a', '0b'],
'ts_range': ["1235", "456"],
'ref_num': 1,
'is_del': 0,
'mtime': 1,

})

blk.add_ref()
blk.add_ref()
self.assertEqual(blk['ref_num'], 3)

blk.rm_ref()
self.assertEqual(blk['ref_num'], 2)

self.assertRaises(ValueError, blk.mark_del)

blk.rm_ref()
blk.rm_ref()
self.assertEqual(blk['ref_num'], 0)
self.assertTrue(blk.can_del())

self.assertRaises(ValueError, blk.rm_ref)

blk.mark_del()
67 changes: 62 additions & 5 deletions ectypes/test/test_block_group.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ def setUp(self):
DriveID('idc000' 'c62d8736c7280002'), 1),
'size': 1000,
'range': ['0a', '0b'],
'ts_range': None,
'ref_num': 1,
'is_del': 0
})

Expand Down Expand Up @@ -150,6 +152,12 @@ def test_mark_delete_block(self):
g = BlockGroup(block_group_id='g000640000000123', idcs=['a', 'b', 'c'], config=_ec_config)

g.add_block(self.foo_block)

self.foo_block.add_ref()
self.assertIsNone(g.mark_delete_block('0000'))
self.assertEqual(self.foo_block['ref_num'], 1)
self.assertEqual(0, self.foo_block['is_del'])

del_blk = g.mark_delete_block('0000')
self.assertDictEqual(del_blk, g.get_block('0000'))

Expand All @@ -161,6 +169,12 @@ def test_mark_delete_block_byid(self):
g = BlockGroup(block_group_id='g000640000000123', idcs=['a', 'b', 'c'], config=_ec_config)

g.add_block(self.foo_block)

self.foo_block.add_ref()
self.assertIsNone(g.mark_delete_block_byid(self.foo_block['block_id']))
self.assertDictEqual(self.foo_block, g.get_block_byid(self.foo_block['block_id']))
self.assertEqual(self.foo_block['ref_num'], 1)

del_blk = g.mark_delete_block_byid(self.foo_block['block_id'])
self.assertDictEqual(del_blk, g.get_block_byid(self.foo_block['block_id']))

Expand All @@ -181,6 +195,10 @@ def test_delete_block(self):
g.add_block(self.foo_block)
self.assertIsNotNone(g.get_block('0000'))

self.foo_block.add_ref()
del_blk = g.delete_block('0000')
self.assertIsNotNone(g.get_block('0000', raise_error=False))

del_blk = g.delete_block('0000')
self.assertIsNone(g.get_block('0000', raise_error=False))
self.assertDictEqual(self.foo_block, del_blk)
Expand Down Expand Up @@ -281,7 +299,9 @@ def test_get_block_idc(self):
DriveID('idc000' 'c62d8736c7280002'), 1),
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
'is_del': 0,
'ref_num': 1,
'ts_range': ["123", "456"]
})
g.add_block(d0)
self.assertEqual('a', g.get_block_idc('0000'))
Expand Down Expand Up @@ -310,7 +330,9 @@ def test_classify_blocks(self):
base_blk = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
'is_del': 0,
'ref_num': 1,
'ts_range': ["123", "456"]
})

ec_blk_idxes = ['0000', '0001']
Expand Down Expand Up @@ -364,7 +386,9 @@ def test_get_parities(self):
base_parity = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
'is_del': 0,
'ref_num': 1,
'ts_range': ["123", "456"]
})

parity_idxes = ['0004', '0005']
Expand Down Expand Up @@ -397,7 +421,11 @@ def make_test_block_group(self, blk_idxes, config=None):
base_blk = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
'ts_range': ["123", "456"],
'ref_num': 1,
'is_del': 0,
'ref_num': 1,
'ts_range': ["123", "456"]
})

if config is None:
Expand Down Expand Up @@ -541,7 +569,7 @@ def test_get_block_byid(self):
self.assertRaises(BlockNotFoundError, bg.get_block_byid, fake_bid, True)
self.assertRaises(BlockNotFoundError, bg.get_block_byid, fake_bid)

def test_delete_block_byid(self):
def test_unlink_block_byid(self):

blk_idxes = ['0000', '0001', '0002', '0003', '0008', '0012']

Expand All @@ -550,6 +578,14 @@ def test_delete_block_byid(self):
blks = bg.indexes_to_blocks(blk_idxes)
bids = [blk['block_id'] for blk in blks]

bg.link_block_byid(bids[1])
self.assertEqual(blks[1]['ref_num'], 2)

self.assertIsNone(bg.delete_block_byid(bids[1]))

del_blk = bg.mark_delete_block_byid(bids[1])
self.assertEqual(del_blk["is_del"], 1)

del_blk = bg.delete_block_byid(bids[1])
self.assertDictEqual(del_blk, blks[1])

Expand All @@ -562,6 +598,27 @@ def test_delete_block_byid(self):

self.assertListEqual(blks, act_blks)

def test_link_block(self):

blk_idxes = ['0000', '0001', '0002', '0003', '0008', '0012']

bg = self.make_test_block_group(blk_idxes)
blks = bg.indexes_to_blocks(blk_idxes)

args = [blks[1], False, False]
self.assertRaises(BlockExists, bg.add_block, *args)
act_blks = bg.indexes_to_blocks(blk_idxes)
self.assertListEqual(blks, act_blks)

bg.link_block(blk_idxes[1])
self.assertEqual(blks[1]['ref_num'], 2)

bg.unlink_block(blk_idxes[1])
self.assertDictEqual(blks[1], bg.add_block(blks[1], allow_exist=True))

act_blks = bg.indexes_to_blocks(blk_idxes)
self.assertListEqual(blks, act_blks)

def test_add_block(self):

blk_idxes = ['0000', '0001', '0002', '0003', '0008', '0012']
Expand Down