diff --git a/ectypes/block_desc.py b/ectypes/block_desc.py index f371d51e..68965bea 100755 --- a/ectypes/block_desc.py +++ b/ectypes/block_desc.py @@ -41,16 +41,53 @@ def _mtime(ts=None): return int(ts) +def _ts_range(ts_range=None): + if ts_range is None: + return ts_range + + if ts_range[0] is not None: + ts_range[0] = str(ts_range[0]) + + if ts_range[1] is not None: + ts_range[1] = str(ts_range[1]) + + return ts_range + + class BlockDesc(FixedKeysDict): keys_default = dict( block_id=_block_id, size=int, range=_range, + ts_range=_ts_range, is_del=_is_del, mtime=_mtime, + ref_num=int, ) def mark_del(self): + if self['ref_num'] != 0: + raise ValueError("cannot mark del block with ref_num:{n} > 0".format( + n=self['ref_num'])) + self["is_del"] = 1 self.mtime = int(time.time()) + + def is_mark_del(self): + return self['is_del'] == 1 + + def add_ref(self): + if self['is_del'] != 0: + raise ValueError("reference a block marked delete") + + self['ref_num'] += 1 + + def rm_ref(self): + if self['ref_num'] < 1: + raise ValueError("ref_num:{n} < 1".format(n=self['ref_num'])) + + self['ref_num'] -= 1 + + def can_del(self): + return self['ref_num'] == 0 diff --git a/ectypes/block_group.py b/ectypes/block_group.py index 6c8171ae..a2f08d70 100755 --- a/ectypes/block_group.py +++ b/ectypes/block_group.py @@ -100,27 +100,55 @@ def make_type_map(self): def mark_delete_block(self, block_index): block = self.get_block(block_index, raise_error=True) - block.mark_del() - return block + block.rm_ref() - def delete_block(self, block_index): - block = self.get_block(block_index, raise_error=True) - del self['blocks'][str(block_index)] + if block.can_del(): + block.mark_del() + return block - return block + return None def mark_delete_block_byid(self, block_id): block = self.get_block_byid(block_id, raise_error=True) - block.mark_del() - return block + block.rm_ref() - def delete_block_byid(self, block_id): + if block.can_del(): + block.mark_del() + return block + + return None + + def unlink_block(self, block_index): + block = self.get_block(block_index, raise_error=True) + + if not block.is_mark_del(): + block.rm_ref() + + if block.can_del(): + del self['blocks'][str(block_index)] + return block + + return None + + def unlink_block_byid(self, block_id): block = self.get_block_byid(block_id, raise_error=True) - del self['blocks'][block_id.block_index] - return block + if not block.is_mark_del(): + block.rm_ref() + + if block.can_del(): + del self['blocks'][block_id.block_index] + return block + + return None + + def delete_block(self, block_index): + return self.unlink_block(block_index) + + def delete_block_byid(self, block_id): + return self.unlink_block_byid(block_id) def has(self, block): bid = block['block_id'] @@ -129,8 +157,23 @@ def has(self, block): existent = self['blocks'].get(bidx) return existent == block + def link_block(self, block_index): + block = self.get_block(block_index, raise_error=True) + + block.add_ref() + return block + + def link_block_byid(self, block_id): + block = self.get_block_byid(block_id, raise_error=True) + + block.add_ref() + return block + def add_block(self, new_block, replace=False, allow_exist=False): + if new_block['ref_num'] != 1: + raise BlockGroupBaseError("the ref_num of block that added must be 1") + if self.has(new_block) and allow_exist: return new_block @@ -251,7 +294,7 @@ def classify_blocks(self, idc_index, only_primary=True): if blk is None: continue - if blk['is_del'] == 1: + if blk.is_mark_del(): mark_del.append(blk) continue diff --git a/ectypes/test/test_block_desc.py b/ectypes/test/test_block_desc.py index eee332fc..668956f9 100755 --- a/ectypes/test/test_block_desc.py +++ b/ectypes/test/test_block_desc.py @@ -19,24 +19,34 @@ def test_blockdesc(self): {'block_id': None, 'size': 0, 'range': None, + 'ts_range': None, + 'ref_num': 0, 'is_del': 0}), ({'block_id': block_id, 'range': ['a', 'b'], + 'ts_range': ["124", None], + 'ref_num': 0, 'size': 34, 'mtime': 1, 'is_del': 0}, {'block_id': BlockID(block_id), 'range': rangeset.Range('a', 'b'), + 'ts_range': ["124", None], + 'ref_num': 0, 'size': 34, 'mtime': 1, 'is_del': 0}), ({'block_id': BlockID(block_id), 'range': rangeset.Range('b', 'bb'), + 'ts_range': ["1235", "456"], + 'ref_num': 0, 'mtime': 1}, {'block_id': BlockID(block_id), 'range': rangeset.Range('b', 'bb'), + 'ts_range': ["1235", "456"], + 'ref_num': 0, 'size': 0, 'mtime': 1, 'is_del': 0, }) @@ -61,15 +71,48 @@ def test_json(self): DriveID('idc000' 'c62d8736c7280002'), 1), 'size': 1000, 'range': ['0a', '0b'], + 'ts_range': ["1235", "456"], + 'ref_num': 1, 'is_del': 0, 'mtime': 1, }) rst = utfjson.dump(blk) - expected = ('{"is_del": 0, "range": ["0a", "0b"], "mtime": 1, "block_id": ' - '"d0g0006400000001230000idc000c62d8736c72800020000000001", "size": 1000}') + + expected = ('{"block_id": "d0g0006400000001230000idc000c62d8736c72800020000000001", "is_del": 0, "ref_num": 1, "range": ["0a", "0b"], "mtime": 1, "ts_range": ["1235", "456"], "size": 1000}') self.assertEqual(expected, rst) loaded = BlockDesc(utfjson.load(rst)) self.assertEqual(blk, loaded) + + def test_ref(self): + block_id = 'd1g0006300000001230101idc000c62d8736c72800020000000001' + blk = BlockDesc({ + 'block_id': block_id, + 'size': 1000, + 'range': ['0a', '0b'], + 'ts_range': ["1235", "456"], + 'ref_num': 1, + 'is_del': 0, + 'mtime': 1, + + }) + + blk.add_ref() + blk.add_ref() + self.assertEqual(blk['ref_num'], 3) + + blk.rm_ref() + self.assertEqual(blk['ref_num'], 2) + + self.assertRaises(ValueError, blk.mark_del) + + blk.rm_ref() + blk.rm_ref() + self.assertEqual(blk['ref_num'], 0) + self.assertTrue(blk.can_del()) + + self.assertRaises(ValueError, blk.rm_ref) + + blk.mark_del() diff --git a/ectypes/test/test_block_group.py b/ectypes/test/test_block_group.py index 904ef316..7fef83ff 100755 --- a/ectypes/test/test_block_group.py +++ b/ectypes/test/test_block_group.py @@ -77,6 +77,8 @@ def setUp(self): DriveID('idc000' 'c62d8736c7280002'), 1), 'size': 1000, 'range': ['0a', '0b'], + 'ts_range': None, + 'ref_num': 1, 'is_del': 0 }) @@ -150,6 +152,12 @@ def test_mark_delete_block(self): g = BlockGroup(block_group_id='g000640000000123', idcs=['a', 'b', 'c'], config=_ec_config) g.add_block(self.foo_block) + + self.foo_block.add_ref() + self.assertIsNone(g.mark_delete_block('0000')) + self.assertEqual(self.foo_block['ref_num'], 1) + self.assertEqual(0, self.foo_block['is_del']) + del_blk = g.mark_delete_block('0000') self.assertDictEqual(del_blk, g.get_block('0000')) @@ -161,6 +169,12 @@ def test_mark_delete_block_byid(self): g = BlockGroup(block_group_id='g000640000000123', idcs=['a', 'b', 'c'], config=_ec_config) g.add_block(self.foo_block) + + self.foo_block.add_ref() + self.assertIsNone(g.mark_delete_block_byid(self.foo_block['block_id'])) + self.assertDictEqual(self.foo_block, g.get_block_byid(self.foo_block['block_id'])) + self.assertEqual(self.foo_block['ref_num'], 1) + del_blk = g.mark_delete_block_byid(self.foo_block['block_id']) self.assertDictEqual(del_blk, g.get_block_byid(self.foo_block['block_id'])) @@ -181,6 +195,10 @@ def test_delete_block(self): g.add_block(self.foo_block) self.assertIsNotNone(g.get_block('0000')) + self.foo_block.add_ref() + del_blk = g.delete_block('0000') + self.assertIsNotNone(g.get_block('0000', raise_error=False)) + del_blk = g.delete_block('0000') self.assertIsNone(g.get_block('0000', raise_error=False)) self.assertDictEqual(self.foo_block, del_blk) @@ -281,7 +299,9 @@ def test_get_block_idc(self): DriveID('idc000' 'c62d8736c7280002'), 1), 'size': 1000, 'range': ['0a', '0b'], - 'is_del': 0 + 'is_del': 0, + 'ref_num': 1, + 'ts_range': ["123", "456"] }) g.add_block(d0) self.assertEqual('a', g.get_block_idc('0000')) @@ -310,7 +330,9 @@ def test_classify_blocks(self): base_blk = BlockDesc({ 'size': 1000, 'range': ['0a', '0b'], - 'is_del': 0 + 'is_del': 0, + 'ref_num': 1, + 'ts_range': ["123", "456"] }) ec_blk_idxes = ['0000', '0001'] @@ -364,7 +386,9 @@ def test_get_parities(self): base_parity = BlockDesc({ 'size': 1000, 'range': ['0a', '0b'], - 'is_del': 0 + 'is_del': 0, + 'ref_num': 1, + 'ts_range': ["123", "456"] }) parity_idxes = ['0004', '0005'] @@ -397,7 +421,11 @@ def make_test_block_group(self, blk_idxes, config=None): base_blk = BlockDesc({ 'size': 1000, 'range': ['0a', '0b'], - 'is_del': 0 + 'ts_range': ["123", "456"], + 'ref_num': 1, + 'is_del': 0, + 'ref_num': 1, + 'ts_range': ["123", "456"] }) if config is None: @@ -541,7 +569,7 @@ def test_get_block_byid(self): self.assertRaises(BlockNotFoundError, bg.get_block_byid, fake_bid, True) self.assertRaises(BlockNotFoundError, bg.get_block_byid, fake_bid) - def test_delete_block_byid(self): + def test_unlink_block_byid(self): blk_idxes = ['0000', '0001', '0002', '0003', '0008', '0012'] @@ -550,6 +578,14 @@ def test_delete_block_byid(self): blks = bg.indexes_to_blocks(blk_idxes) bids = [blk['block_id'] for blk in blks] + bg.link_block_byid(bids[1]) + self.assertEqual(blks[1]['ref_num'], 2) + + self.assertIsNone(bg.delete_block_byid(bids[1])) + + del_blk = bg.mark_delete_block_byid(bids[1]) + self.assertEqual(del_blk["is_del"], 1) + del_blk = bg.delete_block_byid(bids[1]) self.assertDictEqual(del_blk, blks[1]) @@ -562,6 +598,27 @@ def test_delete_block_byid(self): self.assertListEqual(blks, act_blks) + def test_link_block(self): + + blk_idxes = ['0000', '0001', '0002', '0003', '0008', '0012'] + + bg = self.make_test_block_group(blk_idxes) + blks = bg.indexes_to_blocks(blk_idxes) + + args = [blks[1], False, False] + self.assertRaises(BlockExists, bg.add_block, *args) + act_blks = bg.indexes_to_blocks(blk_idxes) + self.assertListEqual(blks, act_blks) + + bg.link_block(blk_idxes[1]) + self.assertEqual(blks[1]['ref_num'], 2) + + bg.unlink_block(blk_idxes[1]) + self.assertDictEqual(blks[1], bg.add_block(blks[1], allow_exist=True)) + + act_blks = bg.indexes_to_blocks(blk_idxes) + self.assertListEqual(blks, act_blks) + def test_add_block(self): blk_idxes = ['0000', '0001', '0002', '0003', '0008', '0012']