Jovian
⭐️
Sign In
In [ ]:
from hangar import Repository
repo = Repository(path = 'C:\Hangar_doc\Repo')
In [ ]:
repo.init(user_name = 'Chinchu', user_email = 'chinchooz007@gmail.com ' , remove_old = True)
In [ ]:
co = repo.checkout(write=True)
co
In [24]:
co.arraysets
Out[24]:
Hangar Arraysets                
    Writeable: True                
    Arrayset Names / Partial Remote References:                
      - 
In [25]:
co.metadata
Out[25]:
Hangar Metadata                
    Writeable: True                
    Number of Keys: 0
In [73]:
import gzip
import pickle
import numpy as np
In [119]:
file_ = gzip.open('C:/Hangar_doc/dataset/mnist.pkl.gz','rb')


train_DS , test_DS , validation_DS = pickle.load(file_ , encoding='bytes')


image_arr = train_DS[0] #image_arr.shape:(50000, 784)
label_arr = train_DS[1] #label_arr.shape:(50000,)
In [128]:
arr_set_img = co.arraysets.init_arrayset(name='train_image', prototype=np.array(image_arr[0]))

# sample value in image_arr--> image_arr[0]-->(shape: (784,)

arr_set_label = co.arraysets.init_arrayset(name='train_label', prototype=np.array(label_arr[0]))

# sample value in label_arr--> label_arr[0]
--------------------------------------------------------------------------- LookupError Traceback (most recent call last) <ipython-input-128-84dea714cd13> in <module> ----> 1 arr_set_img = co.arraysets.init_arrayset(name='train_image', prototype=np.array(image_arr[0])) 2 3 # sample value in image_arr--> image_arr[0]-->(shape: (784,) 4 5 arr_set_label = co.arraysets.init_arrayset(name='train_label', prototype=np.array(label_arr[0])) c:\users\inker_fseai_sys1\appdata\local\programs\python\python36\lib\site-packages\hangar\arrayset.py in init_arrayset(self, name, shape, dtype, prototype, named_samples, variable_shape, backend_opts) 1343 variable_shape=variable_shape) 1344 except (ValueError, LookupError) as e: -> 1345 raise e from None 1346 1347 # ----------- Determine schema format details ------------------------- c:\users\inker_fseai_sys1\appdata\local\programs\python\python36\lib\site-packages\hangar\arrayset.py in init_arrayset(self, name, shape, dtype, prototype, named_samples, variable_shape, backend_opts) 1318 f'Must be <= 64 characters long') 1319 if name in self._arraysets: -> 1320 raise LookupError(f'Arrayset already exists with name: {name}.') 1321 1322 if prototype is not None: LookupError: Arrayset already exists with name: train_image.
In [122]:
co.arraysets
Out[122]:
Hangar Arraysets                
    Writeable: True                
    Arrayset Names / Partial Remote References:                
      - train_image / False
      - train_label / False
In [123]:
arr_set_img
Out[123]:
Hangar ArraysetDataWriter                 
    Arrayset Name             : train_image                
    Schema Hash              : 1=51076e42a840                
    Variable Shape           : False                
    (max) Shape              : (784,)                
    Datatype                 : <class 'numpy.float32'>                
    Named Samples            : True                
    Access Mode              : a                
    Number of Samples        : 0                
    Partial Remote Data Refs : False
In [124]:
arr_set_label

Out[124]:
Hangar ArraysetDataWriter                 
    Arrayset Name             : train_label                
    Schema Hash              : 1=2a5b812f0f20                
    Variable Shape           : False                
    (max) Shape              : ()                
    Datatype                 : <class 'numpy.int64'>                
    Named Samples            : True                
    Access Mode              : a                
    Number of Samples        : 0                
    Partial Remote Data Refs : False
In [125]:
repo.log()
In [126]:
repo.list_branches()
Out[126]:
['master']
In [127]:
import time
time_begin = time.time()
for index, image in enumerate(image_arr):
    arr_set_img.add(data=image, name=index)
    arr_set_labels.add(data=np.array([label_arr[index]]), name=str(index))
time_end = time.time()
print("Non-context manager completed in {} seconds".format(time_end - time_begin))
--------------------------------------------------------------------------- KeyError Traceback (most recent call last) <ipython-input-127-53d987399cf0> in <module> 2 time_begin = time.time() 3 for index, image in enumerate(image_arr): ----> 4 arr_set_img.add(data=image, name=index) 5 arr_set_labels.add(data=np.array([label_arr[index]]), name=str(index)) 6 time_end = time.time() c:\users\inker_fseai_sys1\appdata\local\programs\python\python36\lib\site-packages\hangar\arrayset.py in add(self, data, name, **kwargs) 781 existingHashVal = self._hashTxn.get(hashKey, default=False) 782 if existingHashVal is False: --> 783 hashVal = self._fs[self._dflt_backend].write_data(data) 784 self._hashTxn.put(hashKey, hashVal) 785 self._stageHashTxn.put(hashKey, hashVal) c:\users\inker_fseai_sys1\appdata\local\programs\python\python36\lib\site-packages\hangar\backends\hdf5_00.py in write_data(self, array, remote_operation) 796 destSlc = (self.slcExpr[self.hIdx], self.slcExpr[0:array.size]) 797 flat_arr = np.ravel(array) --> 798 self.wFp[self.w_uid][f'/{self.hNextPath}'].write_direct(flat_arr, srcSlc, destSlc) 799 800 hashVal = hdf5_00_encode(uid=self.w_uid, h5py\_objects.pyx in h5py._objects.with_phil.wrapper() h5py\_objects.pyx in h5py._objects.with_phil.wrapper() c:\users\inker_fseai_sys1\appdata\local\programs\python\python36\lib\site-packages\h5py\_hl\group.py in __getitem__(self, name) 260 raise ValueError("Invalid HDF5 object reference") 261 else: --> 262 oid = h5o.open(self.id, self._e(name), lapl=self._lapl) 263 264 otype = h5i.get_type(oid) h5py\_objects.pyx in h5py._objects.with_phil.wrapper() h5py\_objects.pyx in h5py._objects.with_phil.wrapper() h5py\h5o.pyx in h5py.h5o.open() KeyError: "Unable to open object (object '0' doesn't exist)"
In [132]:

from hangar import Repository
import numpy as np
repos = Repository(path = 'C:\Repo')
repos.init(user_name= 'Chinchu', user_email= 'chinchooz007@gmail.com', remove_old = True)
co = repos.checkout(write=True)
c:\users\inker_fseai_sys1\appdata\local\programs\python\python36\lib\site-packages\hangar\context.py:222: UserWarning: No repository exists at C:\Repo\.hangar, please use `repo.init()` method warnings.warn(msg, UserWarning)
Hangar Repo initialized at: C:\Repo\.hangar
In [140]:
random_arr = np.random.randint(1, 10, 10)
arr_set = co.arraysets.init_arrayset(name='random_arrayset_', prototype=random_arr)
arr_set['0'] = random_arr

--------------------------------------------------------------------------- OSError Traceback (most recent call last) <ipython-input-140-21ffb234dab3> in <module> 1 random_arr = np.random.randint(1, 10, 10) 2 arr_set = co.arraysets.init_arrayset(name='random_arrayset_', prototype=random_arr) ----> 3 arr_set['0'] = random_arr c:\users\inker_fseai_sys1\appdata\local\programs\python\python36\lib\site-packages\hangar\arrayset.py in __setitem__(self, key, value) 567 sample name of the stored data (assuming operation was successful) 568 """ --> 569 self.add(value, key) 570 return key 571 c:\users\inker_fseai_sys1\appdata\local\programs\python\python36\lib\site-packages\hangar\arrayset.py in add(self, data, name, **kwargs) 781 existingHashVal = self._hashTxn.get(hashKey, default=False) 782 if existingHashVal is False: --> 783 hashVal = self._fs[self._dflt_backend].write_data(data) 784 self._hashTxn.put(hashKey, hashVal) 785 self._stageHashTxn.put(hashKey, hashVal) c:\users\inker_fseai_sys1\appdata\local\programs\python\python36\lib\site-packages\hangar\backends\numpy_10.py in write_data(self, array, remote_operation) 409 self._create_schema(remote_operation=remote_operation) 410 else: --> 411 self._create_schema(remote_operation=remote_operation) 412 413 destSlc = (self.slcExpr[self.hIdx], *(self.slcExpr[0:x] for x in array.shape)) c:\users\inker_fseai_sys1\appdata\local\programs\python\python36\lib\site-packages\hangar\backends\numpy_10.py in _create_schema(self, remote_operation) 325 else: 326 symlink_file_path = pjoin(self.STAGEDIR, f'{uid}.npy') --> 327 symlink_rel(file_path, symlink_file_path) 328 329 def read_data(self, hashVal: NUMPY_10_DataHashSpec) -> np.ndarray: c:\users\inker_fseai_sys1\appdata\local\programs\python\python36\lib\site-packages\hangar\utils.py in symlink_rel(src, dst, is_dir) 115 """ 116 rel_path_src = os.path.relpath(src, os.path.dirname(dst)) --> 117 os.symlink(rel_path_src, dst, target_is_directory=is_dir) 118 119 OSError: symbolic link privilege not held
In [141]:
co.close()

In [ ]: