Commit 7c0a8117 authored by M. Huang's avatar M. Huang
Browse files

live shape and image for ArrayDataset. get_sample_prod sorted; odict gives KeyError

parent 449cc180
__version_info__ = (1, 6, 5)
__version_info__ = (1, 6, 6)
__version__ = '.'.join(map(str, __version_info__))
......@@ -5,7 +5,7 @@ from .typed import Typed
from .typecoded import Typecoded
from .listener import ColumnListener
from .ndprint import ndprint
from ..utils.common import mstr, bstr, lls, exprstrs
from ..utils.common import mstr, bstr, lls, exprstrs, findShape
from .dataset import GenericDataset
try:
from .arraydataset_datamodel import Model
......@@ -43,12 +43,14 @@ class ArrayDataset(GenericDataset, Iterable):
""" Initializes an ArrayDataset.
Default ```None``` will initialize MetaData Parameters to their default values.
If ``data`` is not None and has shape (``len`` applies), ``shape`` MDP is set to the actual ``data`` shape. If ``data`` is not given but ``shape`` is given, ``shape`` MDP is set to ``shape``.
"""
# collect MDPs from args-turned-local-variables.
metasToBeInstalled = OrderedDict(
itertools.filterfalse(
lambda x: x[0] in ('self', '__class__', 'zInfo', 'kwds'),
lambda x: x[0] in ('self', '__class__',
'zInfo', 'kwds', 'shape'),
locals().items())
)
......@@ -58,9 +60,10 @@ class ArrayDataset(GenericDataset, Iterable):
# print('@1 zInfo', id(self.zInfo['metadata']), id(self), id(self.zInfo),
# self.zInfo['metadata']['version'], list(metasToBeInstalled.keys()))
# must be the first line to initiate meta
super().__init__(zInfo=zInfo, **metasToBeInstalled, **kwds)
dshape = findShape(data)
self.shape = dshape if dshape else shape
# def getData(self):
# """ Optimized """
......@@ -72,13 +75,21 @@ class ArrayDataset(GenericDataset, Iterable):
isitr = hasattr(data, '__iter__') # and hasattr(data, '__next__')
if not isitr and data is not None:
# dataWrapper initializes data as None
m = 'data in ArrayDataset must be a subclass of Sequence: ' + \
m = 'data in ArrayDataset must be an iterator, not ' + \
data.__class__.__name__
raise TypeError(m)
d = None if data is None else \
data if hasattr(data, '__getitem__') else list(data)
# no passive shape-updating. no
self.shape = findShape(d)
super(ArrayDataset, self).setData(d)
def updateShape(self):
shape = findShape(self.data)
self.shape = shape
return shape
def __setitem__(self, *args, **kwargs):
""" sets value at key.
"""
......
......@@ -2,6 +2,8 @@
from collections import ChainMap
import os
from copy import copy
from .metadataholder import MetaDataHolder
from .metadata import AbstractParameter, Parameter
from .datatypes import DataTypes
......@@ -18,6 +20,10 @@ MdpInfo = {}
# These property names are not for mormal properties
Reserved_Property_Names = ['history', 'meta', 'refs', 'dataset',
'zInfo', '_MDP', 'extraMdp', 'alwaysMeta']
# these MDPs and vital attrbutes are Set By Parent classes:
# meta via attributable, description annotatable,
# type typed, data and shape dataWrapper
MDP_Set_by_Parents = ['meta', 'description', 'type', 'data']
class Attributable(MetaDataHolder):
......@@ -48,11 +54,16 @@ class Attributable(MetaDataHolder):
_x = 'typ_' if x == 'type' else x
if _x not in kwds:
mdps[x] = zm[x]['default']
else:
elif _x not in MDP_Set_by_Parents:
kwval = kwds.pop(_x)
mdps[x] = zm[x]['default'] if kwval is None else kwval
self._MDP = ChainMap(self.extraMdp, zm)
# This will set MDP_Set_by_Parents args.
super(Attributable, self).__init__(meta=meta, **kwds)
# do not set MDPs that have been set by super classes
for p in copy(mdps):
if p in self._meta:
mdps.pop(p)
self.setParameters(mdps)
def setParameters(self, params):
......
# -*- coding: utf-8 -*-
from collections import OrderedDict, UserDict
from collections.abc import Collection
from .serializable import Serializable
from .eq import DeepEqual, xhash
from ..utils.common import bstr
from ..utils.ydump import ydump
from collections import OrderedDict, UserDict
from pprint import pformat
import logging
......@@ -23,10 +22,9 @@ class ODict(UserDict, Serializable, DeepEqual):
"""
"""
# print(args)
# data = OrderedDict(*args, **kwds)
super().__init__(*args, **kwds)
# UserDict.__init__(self, data)
self.__missing__ = None
Serializable.__init__(self)
# @property
......@@ -86,6 +84,17 @@ class ODict(UserDict, Serializable, DeepEqual):
OD_toString_Nest -= 1
return d + '>'
def get(self, name):
""" Raise a ``KeyErrpr`` to change the default behavior of colections.Mapping to quietly return a None when a key is not found in the dict.
"""
return self.data[name]
res = super().__getitem__(name)
if res is not None or name in self.data:
return res
logger.debug('%s is not found in %s.' % (name, self))
raise KeyError()
# def __repr__(self):
# """ returns string representation with details set according to debuglevel.
# """
......
......@@ -3,7 +3,7 @@
from .indexed import Indexed
from .ndprint import ndprint
from .odict import ODict
from ..utils.common import mstr, bstr, lls, exprstrs
from ..utils.common import mstr, bstr, lls, exprstrs, findShape
from .dataset import Dataset
from .indexed import Indexed
......@@ -358,9 +358,13 @@ class TableDataset(Dataset, TableModel):
for x in self.getData().values():
ret.append(x.data[rowIndex])
del x.data[rowIndex]
x.shape = findShape(x)
return ret
return [x.data.pop(rowIndex) for x in self.getData().values()]
r = []
for x in self.getData().values():
r.append(x.data.pop(rowIndex))
x.shape = findShape(x)
return r
@ property
def rowCount(self):
......
from fdi.dataset.product import _Model_Spec as PPI
from .product import Product
from .numericparameter import NumericParameter
from .dateparameter import DateParameter
from .stringparameter import StringParameter
from .datatypes import Vector
from .dataset import CompositeDataset
......@@ -10,6 +11,8 @@ from ..pal.context import Context, MapContext
from .finetime import FineTime
import copy
from math import sin, cos
from os import path as op
class TP(Product):
......@@ -62,38 +65,62 @@ class SP(Product):
def get_sample_product():
"""
A complex product as a reference for testing and demo.
"""
prodx = Product('A complex product')
prodx.creator = 'Frankenstein'
# add a parameter with validity descriptors to the product
prodx.meta['speed'] = NumericParameter(
description='an extra param',
value=Vector((1.1, 2.2, 3.3)),
valid={(1, 22): 'normal', (30, 33): 'fast'}, unit='meter')
# an arraydsets
a1 = [768, 767, 766, 4.4, 4.5, 4.6, 5.4E3]
a2 = 'C'
a3 = 'Temperature'
a4 = ArrayDataset(data=a1, unit=a2, description='An Array')
# metadata to the dataset
a11 = 'T0'
a12 = DateParameter('2020-02-02T20:20:20.0202',
description='meta of composite')
# This is not the best as a4.T0 does not exist
# a4.meta[a11] = a12
# this does it a4.T0 = a12 or:
setattr(a4, a11, a12)
# put the arraydataset to the product with a name a3.
prodx[a3] = a4
compo = CompositeDataset()
# two arraydsets
a1 = [768, 4.4, 5.4E3]
a2 = 'ev'
a3 = 'arraydset 1'
a4 = ArrayDataset(data=a1, unit=a2, description=a3)
a5, a6, a7 = [[1.09, 289], [3455, 564]
], 'count', 'background -- arraydset in compo'
a8 = ArrayDataset(data=a5, unit=a6, description=a7)
a10 = 'calibration_arraydset'
prodx['results'] = compo
a5 = [[109, 289, 9], [88, 3455, 564]]
a8 = ArrayDataset(data=a5, unit='count', description='array in composite')
a10 = 'calibration'
# put the dataset to the compositedataset. here set() api is used
compo.set(a10, a8)
# a tabledataset
ELECTRON_VOLTS = 'eV'
SECONDS = 'sec'
t = [x * 1.0 for x in range(5)]
METERS = 'm'
t = [x * 1.0 for x in range(9)]
e = [2 * x + 100 for x in t]
x = TableDataset(description="Example table")
y = [10 * sin(x*2*3.14/len(t)) for x in t]
z = [10 * cos(x*2*3.14/len(t)) for x in t]
x = TableDataset(description="A table")
x["Time"] = Column(data=t, unit=SECONDS)
x["Energy"] = Column(data=e, unit=ELECTRON_VOLTS)
x["y"] = Column(data=y, unit=METERS)
x["z"] = Column(data=z, unit=METERS)
# set a tabledataset ans an arraydset, with a parameter in metadata
a13 = 'energy_table'
# metadata to the dataset
compo[a13] = x
a11 = 'm1'
a12 = StringParameter('EX')
compo.meta[a11] = a12
prodx = Product('complex prod')
prodx.meta['extra'] = NumericParameter(description='a different param in metadata',
value=Vector((1.1, 2.2, 3.3)), valid={(1, 22): 'normal', (30, 33): 'fast'}, unit='meter')
prodx[a3] = a4
prodx['results'] = compo
compo['Time_Energy_Pos'] = x
# an image
fname = 'imageBlue.png'
with open(op.abspath(op.dirname(__file__))+'/resources/'+fname, 'rb') as f:
image = ArrayDataset(data=f.read(), description='An image in an array')
image.file = fname
image.content = 'Content-type: image/png'
prodx['Image'] = image
return prodx
......@@ -506,3 +506,22 @@ def getUidGid(username):
logger.error(msg)
return uid, gid
def findShape(data):
"""
"""
if data is None:
return None
shape = []
d = data
while d is not None:
if issubclass(d.__class__, (str)):
d = None
else:
try:
shape.append(len(d))
d = d[0]
except (TypeError, IndexError) as e:
d = None
return tuple(shape)
......@@ -19,20 +19,21 @@ If the above fail and a method whose name starts with 'is' then the method is ca
paths = paths.split('/')
p0 = paths[0]
found_meth = None
found_method = None
is_str = issubclass(p0.__class__, str)
if is_str and hasattr(nested, p0):
v = getattr(nested, p0)
rep = re + '.'+p0
if inspect.ismethod(v) and p0.startswith('is'):
found_meth = v
found_method = v
else:
if len(paths) == 1:
return v, rep
return fetch(paths[1:], v, rep)
else:
if is_str and ',' in p0:
# p0 is a set of arguments of int and float
num = []
for seg in p0.split(','):
try:
......@@ -56,8 +57,8 @@ If the above fail and a method whose name starts with 'is' then the method is ca
except TypeError:
pass
# not attribute or member
if found_meth:
if found_method:
# return methodcaller(p0)(nested), rep + '()'
return found_meth(), rep + '()'
return found_method(), rep + '()'
return None, '%s has no attribute or member: %s.' % (re, p0)
......@@ -227,14 +227,14 @@ meta= {
=============== ==================== ====== ======== ==================== ================= ====== =====================
name value unit type valid default code description
=============== ==================== ====== ======== ==================== ================= ====== =====================
shape (2, 3, 4, 5) tuple None () Number of elements in
each dimension. Quic
k changers to the rig
ht.
description toString tester AD string None UNKNOWN B Description of this d
ataset
unit lyr string None None B Unit of every element
.
shape () tuple None () Number of elements in
each dimension. Quic
k changers to the rig
ht.
typecode UNKNOWN string None UNKNOWN B Python internal stora
ge code.
version 0.1 string None 0.1 B Version of dataset
......@@ -302,15 +302,15 @@ ArrayDataset-dataset =
level 1
=== ArrayDataset () ===
meta= {
-------------------------- -------------------------- ----------------
description= toString test unit= lyr shape= ()
er AD
typecode= UNKNOWN version= 0.1 FORMATV= 1.6.0.1
a= 3.4 b= xy (2019-02-19T01:02:03 c= Invalid (IJK)
.456789
1929229360456789)
d= off (0b00) added_parameter= 42
-------------------------- -------------------------- ----------------
------------------- -------------------------- ----------------
shape= (2, 3, 4, 5) description= toString test unit= lyr
er AD
typecode= UNKNOWN version= 0.1 FORMATV= 1.6.0.1
a= 3.4 b= xy (2019-02-19T01:02:03 c= Invalid (IJK)
.456789
1929229360456789)
d= off (0b00) added_parameter= 42
------------------- -------------------------- ----------------
MetaData-listeners = ListnerSet{}
}
ArrayDataset-dataset =
......@@ -358,10 +358,10 @@ ArrayDataset-dataset =
level 2, repr
ArrayDataset(description=toString tester AD, unit=lyr, a=3.4, b=xy (FineTime(2019-02-19T01:02:03.456789)), c=Invalid (IJK), d=off (0b00), added_parameter=42. data= [[[[0, 0, 0, ...0]]], [[[0, 0, 0, ...0]]]])
ArrayDataset(shape=(2, 3, 4, 5), description=toString tester AD, unit=lyr, a=3.4, b=xy (FineTime(2019-02-19T01:02:03.456789)), c=Invalid (IJK), d=off (0b00), added_parameter=42. data= [[[[0, 0, 0, ...0]]], [[[0, 0, 0, ...0]]]])
an empty meta and long data level 2:
ArrayDataset(Default Meta. data= [8, 8, 8, 8, 8, 8, 8, 8])
ArrayDataset(shape=(8,). data= [8, 8, 8, 8, 8, 8, 8, 8])
"""
out_TableDataset = """
......@@ -427,7 +427,7 @@ TableDataset-dataset =
level 2, repr
TableDataset(a=3.4, b=xy (FineTime(2019-02-19T01:02:03.456789)), c=Invalid (IJK), d=off (0b00), added_parameter=42.data= {"col1": Column(description=1, unit=eV. data= [1, 4.4, 5400.0]), "col2": Column(description=2, unit=cnt. data= [0, 43.2, 2000.0])})
TableDataset(a=3.4, b=xy (FineTime(2019-02-19T01:02:03.456789)), c=Invalid (IJK), d=off (0b00), added_parameter=42.data= {"col1": Column(shape=(3,), description=1, unit=eV. data= [1, 4.4, 5400.0]), "col2": Column(shape=(3,), description=2, unit=cnt. data= [0, 43.2, 2000.0])})
an empty level 2:
TableDataset(Default Meta.data= {})
......@@ -463,14 +463,14 @@ meta= {
=========== =========== ====== ====== ======= ========= ====== =====================
name value unit type valid default code description
=========== =========== ====== ====== ======= ========= ====== =====================
shape (3,) tuple None () Number of elements in
each dimension. Quic
k changers to the rig
ht.
description arraydset 1 string None UNKNOWN B Description of this d
ataset
unit ev string None None B Unit of every element
.
shape () tuple None () Number of elements in
each dimension. Quic
k changers to the rig
ht.
typecode UNKNOWN string None UNKNOWN B Python internal stora
ge code.
version 0.1 string None 0.1 B Version of dataset
......@@ -519,10 +519,10 @@ MetaData-listeners = ListnerSet{}
CompositeDataset-datasets =
<ODict === ArrayDataset () ===
meta= {
------------------------ ------------ ----------------
description= arraydset 1 unit= ev shape= ()
typecode= UNKNOWN version= 0.1 FORMATV= 1.6.0.1
------------------------ ------------ ----------------
----------------- ------------------------ ----------------
shape= (3,) description= arraydset 1 unit= ev
typecode= UNKNOWN version= 0.1 FORMATV= 1.6.0.1
----------------- ------------------------ ----------------
MetaData-listeners = ListnerSet{}
}
ArrayDataset-dataset =
......@@ -550,7 +550,7 @@ TableDataset-dataset =
meta
CompositeDataset-datasets =
<ODict ArrayDataset(description=arraydset 1, unit=ev. data= [768, 4.4, 5400.0]) TableDataset(description=Example table.data= {"Time": Column(unit=sec. data= [0.0, 1.0, 2.0, 3.0, 4.0]), "Energy": Column(unit=eV. data= [100.0, 102.0, 104.0, 106.0, 108.0])})>"""
<ODict ArrayDataset(shape=(3,), description=arraydset 1, unit=ev. data= [768, 4.4, 5400.0]) TableDataset(description=Example table.data= {"Time": Column(shape=(5,), unit=sec. data= [0.0, 1.0, 2.0, 3.0, 4.0]), "Energy": Column(shape=(5,), unit=eV. data= [100.0, 102.0, 104.0, 106.0, 108.0])})>"""
out_FineTime = """toString test
=========== format: "%Y-%m-%dT%H:%M:%S.%f" =======
FineTime
......
......@@ -45,7 +45,7 @@ from fdi.dataset.history import History
from fdi.dataset.baseproduct import BaseProduct
from fdi.dataset.product import Product
from fdi.dataset.readonlydict import ReadOnlyDict
from fdi.dataset.testproducts import SP
from fdi.dataset.testproducts import SP, get_sample_product
from fdi.utils.checkjson import checkjson
# import __builtins__
......@@ -1129,6 +1129,24 @@ def test_Attributable():
v = Attributable(meta=a1)
assert v.getMeta() == a1
# non-MDP attributable
a3 = 'Temperature'
a4 = ArrayDataset(data=[1, 2], unit='C', description='An Array')
# metadata to the dataset
a11 = 'T0'
a12 = DateParameter('2020-02-02T20:20:20.0202',
description='meta of composite')
# This is not the best as a4.T0 does not exist
a4.meta[a11] = a12
with pytest.raises(AttributeError):
assert a4.T0 == a12.value
# this does it
setattr(a4, a11, a12)
assert a4.T0 == a12.value
# or
a4.T0a = a12
assert a4.T0a == a12.value
checkgeneral(v)
......@@ -1230,13 +1248,19 @@ def do_ArrayDataset_init(atype):
assert v.unit == a2
assert v.description == a3
assert v.typecode == a6
assert v.shape == a7
assert v.shape == (len(a1),)
v = ArrayDataset(data=a1)
assert v.data == a1
assert v.unit is None
assert v.description == 'UNKNOWN'
assert v.typecode == 'UNKNOWN'
assert v.shape == ()
assert v.shape == (len(a1),)
ashape = [[[1], [2]], [[3], [4]], [[5], [6]]]
v2 = ArrayDataset(data=ashape)
assert v2.shape == (3, 2, 1)
v2 = ArrayDataset(data=['poi'])
assert v2.shape == (1,)
# omit the parameter names when instantiating, the orders are data, unit, description
v2 = ArrayDataset(a1)
......@@ -1530,6 +1554,7 @@ def test_TableDataset_func_row():
assert v.removeRow(1) == [c1.data[1], c2.data[1]]
assert v.rowCount == 4
assert cc.pop(1) == c1.data[1]
cc.updateShape() # not done automatically
assert v['col3'] == cc
# read rowa with slice
s = v.getRow(slice(1, 3))
......@@ -1540,6 +1565,7 @@ def test_TableDataset_func_row():
assert v.removeRow(slice(1, 3)) == [[3.3, 6.6], [4.4, 8.8]]
assert v.rowCount == 2
del cc[1:3]
cc.updateShape() # not done automatically
assert v['col3'] == cc
# acess rows
......@@ -2245,6 +2271,13 @@ def test_FineTimes_toString():
assert ts == out_FineTime
def test_get_sample_product():
v = get_sample_product()
assert v['Image'].data[1:4] == b'PNG'
checkjson(v)
checkgeneral(v)
def test_History():
v = History()
checkjson(v)
......@@ -2315,8 +2348,8 @@ def test_BaseProduct():
assert x.creator == a1
# normal metadata
# if the attitute does not exist, return None. This is an OrderedDict behavior.
assert x.meta['notthere'] is None
with pytest.raises(KeyError):
x.meta['notthere']
# test comparison:
p1 = BaseProduct(description="oDescription")
......
......@@ -818,9 +818,9 @@ def doquery(poolpath, newpoolpath):
chk(res[2], rec1[5])
chk(res[3], rec1[6])
# all 'time' < 5006. will cause TypeError because some Contex data do not have 'time'
# all 'time' < 5006. will cause KeyError because some Contex data do not have 'time'
q = MetaQuery(Context, 'm["time"] < 5006')
with pytest.raises(TypeError):
with pytest.raises(KeyError):
res = pstore.select(q)
# all 'time' < 5006 mapcontext. all in newpool
......
......@@ -3,6 +3,7 @@
from fdi.utils.leapseconds import utc_to_tai, tai_to_utc, dTAI_UTC_from_utc, _fallback
from fdi.dataset.eq import deepcmp
from fdi.dataset.metadata import make_jsonable
from fdi.dataset.finetime import FineTime
from fdi.dataset.datatypes import Vector, Quaternion
from fdi.dataset.deserialize import Class_Look_Up
from fdi.dataset.testproducts import get_sample_product
......@@ -122,60 +123,74 @@ def test_fetch():
v, s = fetch(["description"], p)
assert v == p.description
assert s == '.description'
# metadata
e = p.meta['extra']
v, s = fetch(["meta", "extra"], p)
assert v == p.meta['extra']
assert s == '.meta["extra"]'
# metadatax
with pytest.raises(KeyError):
e = p.meta['extra']
e = p.meta['speed']
v, s = fetch(["meta", "speed"], p)
assert v == p.meta['speed']
assert s == '.meta["speed"]'
# parameter
v, s = fetch(["meta", "extra", "unit"], p)
v, s = fetch(["meta", "speed", "unit"], p)
assert v == 'meter'
assert v == p.meta['extra'].unit
assert s == '.meta["extra"].unit'
assert v == p.meta['speed'].unit
assert s == '.meta["speed"].unit'
v, s = fetch(["meta", "extra", "value"], p)
v, s = fetch(["meta", "speed", "value"], p)
assert v == Vector((1.1, 2.2, 3.3))
assert v == p.meta['extra'].value
assert s == '.meta["extra"].value'
v, s = fetch(["meta", "extra", "valid"], p)
assert v == p.meta['speed'].value
assert s == '.meta["speed"].value'
v, s = fetch(["meta", "speed", "valid"], p)
mkj = make_jsonable({(1, 22): 'normal', (30, 33): 'fast'})
assert v == mkj
assert v == make_jsonable(p.meta['extra'].valid)
assert s == '.meta["extra"].valid'
assert v == make_jsonable(p.meta['speed'].valid)
assert s == '.meta["speed"].valid'
# TODO written is string
# [[[1, 22], 'normal'], [[30, 33], 'fast']]
v, s = fetch(["meta", "extra", "valid", 0, 1], p)
v, s = fetch(["meta", "speed", "valid", 0, 1], p)
assert v == 'normal'
assert v == p.meta['extra'].valid[0][1]
assert s == '.meta["extra"].valid[0][1]'
assert v == p.meta['speed'].valid[0][1]
assert s == '.meta["speed"].valid[0][1]'
#
# validate execution
v, s = fetch(["meta", "extra", "isValid", ], p)
v, s = fetch(["meta", "speed", "isValid", ], p)
assert v == True
assert v == p.meta['extra'].isValid()
assert s == '.meta["extra"].isValid()'
# datasets
v, s = fetch(["arraydset 1", "unit"], p)
assert v == 'ev'
assert v == p['arraydset 1'].unit
assert s == '["arraydset 1"].unit'
assert v == p.meta['speed'].isValid()