Commit 9c713451 authored by M. Huang's avatar M. Huang
Browse files

1.5.5 implement MDP; fix bugs in test

parent d7655dda
docs/sphinx/_static/classes_dataset.png

1.16 MB | W: | H:

docs/sphinx/_static/classes_dataset.png

1.43 MB | W: | H:

docs/sphinx/_static/classes_dataset.png
docs/sphinx/_static/classes_dataset.png
docs/sphinx/_static/classes_dataset.png
docs/sphinx/_static/classes_dataset.png
  • 2-up
  • Swipe
  • Onion skin
docs/sphinx/_static/classes_pal.png

468 KB | W: | H:

docs/sphinx/_static/classes_pal.png

697 KB | W: | H:

docs/sphinx/_static/classes_pal.png
docs/sphinx/_static/classes_pal.png
docs/sphinx/_static/classes_pal.png
docs/sphinx/_static/classes_pal.png
  • 2-up
  • Swipe
  • Onion skin
docs/sphinx/_static/packages_pal.png

99.8 KB | W: | H:

docs/sphinx/_static/packages_pal.png

114 KB | W: | H:

docs/sphinx/_static/packages_pal.png
docs/sphinx/_static/packages_pal.png
docs/sphinx/_static/packages_pal.png
docs/sphinx/_static/packages_pal.png
  • 2-up
  • Swipe
  • Onion skin
docs/sphinx/_static/packages_pns.png

16.6 KB | W: | H:

docs/sphinx/_static/packages_pns.png

21.7 KB | W: | H:

docs/sphinx/_static/packages_pns.png
docs/sphinx/_static/packages_pns.png
docs/sphinx/_static/packages_pns.png
docs/sphinx/_static/packages_pns.png
  • 2-up
  • Swipe
  • Onion skin
......@@ -18,7 +18,10 @@ class Annotatable(object):
def description(self):
""" xx must be a property for ``self.xx = yy`` to work in super class after xx is set as a property also by a subclass.
"""
return self._description
try:
return self._description
except AttributeError:
return None
@description.setter
def description(self, description):
......
......@@ -2,38 +2,65 @@
from .datawrapper import DataWrapper
from .typed import Typed
from .typecoded import Typecoded
from .listener import ColumnListener
from .ndprint import ndprint
from ..utils.common import mstr, bstr, lls, exprstrs
from .dataset import GenericDataset
try:
from poperties_NumericParameter import MetaDataProperties
from .arraydataset_datamodel import Model
except ImportError:
class MetaDataProperties():
pass
Model = {'metadata': {}}
from collections.abc import Sequence
from collections import OrderedDict
import itertools
import logging
# create logger
logger = logging.getLogger(__name__)
#logger.debug('level %d' % (logger.getEffectiveLevel()))
# logger.debug('level %d' % (logger.getEffectiveLevel()))
MdpInfo = Model['metadata']
class ArrayDataset(MetaDataProperties, DataWrapper, GenericDataset, Sequence, Typed):
class ArrayDataset(DataWrapper, GenericDataset, Sequence, Typed, Typecoded):
""" Special dataset that contains a single Array Data object.
mh: If omit the parameter names during instanciation, e.g. ArrayDataset(a, b, c), the assumed order is data, unit, description.
mh: contains a sequence which provides methods count(), index(), remove(), reverse().
A mutable sequence would also need append(), extend(), insert(), pop() and sort().
"""
def __init__(self, data=None, unit=None, description='UNKNOWN', typ_=None, default=None, **kwds):
def __init__(self, data=None,
unit=None,
description=None,
typ_=None,
typecode=None,
zInfo=None,
alwaysMeta=True,
** kwds):
""" Initializes an ArrayDataset.
Default ```None``` will initialize MetaData Parameters to their default values.
"""
self.setDefault(default)
super(ArrayDataset, self).__init__(data=data, unit=unit,
description=description, typ_=typ_, **kwds) # initialize data, meta
# collect MDPs from args-turned-local-variables.
metasToBeInstalled = OrderedDict(
itertools.filterfalse(
lambda x: x[0] in ('self', '__class__', 'zInfo', 'kwds'),
locals().items())
)
global Model
if zInfo is None:
zInfo = Model
# print('@1 zInfo', id(self.zInfo['metadata']), id(self), id(self.zInfo),
# self.zInfo['metadata']['version'], list(metasToBeInstalled.keys()))
# must be the first line to initiate meta
super().__init__(zInfo=zInfo, **metasToBeInstalled, **kwds)
# def getData(self):
# """ Optimized """
......@@ -52,24 +79,6 @@ class ArrayDataset(MetaDataProperties, DataWrapper, GenericDataset, Sequence, Ty
data if hasattr(data, '__getitem__') else list(data)
super(ArrayDataset, self).setData(d)
@property
def default(self):
return self.getDefault()
@default.setter
def default(self, default):
self.setDefault(default)
def getDefault(self):
""" Returns the default related to this object."""
return self._default
def setDefault(self, default):
""" Sets the default of this object.
"""
self._default = default
def __setitem__(self, *args, **kwargs):
""" sets value at key.
"""
......@@ -128,13 +137,18 @@ class ArrayDataset(MetaDataProperties, DataWrapper, GenericDataset, Sequence, Ty
if level > 1:
vs, us, ts, ds, fs, gs, cs = exprstrs(self, '_data')
return cn +\
'{ %s (%s) <%s>, "%s", default %s, tcode=%s, meta=%s}' %\
(vs, us, ts, ds, fs, cs, str(self.meta))
return cn + '(%s ' % vs + \
', '.join(str(k)+': '+str(v.value if hasattr(v, 'value') else v)
for k, v in self.meta.items()) + ')'
# '{ %s (%s) <%s>, "%s", default %s, tcode=%s}' %\
#(vs, us, ts, ds, fs, cs)
s = '=== %s (%s) ===\n' % (cn, self.description if hasattr(
self, 'descripion') else '')
s += mstr(self.__getstate__(), level=level,
toshow = {'meta': self.__getstate__()['meta'],
'data': self.__getstate__()['data'],
}
s += mstr(toshow, level=level,
tablefmt=tablefmt, tablefmt1=tablefmt1, tablefmt2=tablefmt2,
excpt=['description'], **kwds)
......@@ -149,12 +163,13 @@ class ArrayDataset(MetaDataProperties, DataWrapper, GenericDataset, Sequence, Ty
""" Can be encoded with serializableEncoder """
# s = OrderedDict(description=self.description, meta=self.meta, data=self.data) # super(...).__getstate__()
s = OrderedDict(description=self.description,
meta=self._meta,
meta=self._meta if hasattr(self, '_meta') else None,
data=None if self.data is None else self.data,
type=self._type,
unit=self._unit,
default=self._default,
typecode=self._typecode,
type=self.type,
unit=self.unit,
typecode=self.typecode,
version=self.version,
FORMATV=self.FORMATV,
_STID=self._STID)
return s
......
# -*- coding: utf-8 -*-
# Automatically generated from fdi/dataset/resources/ArrayDataset_DataModel.yml. Do not edit.
from collections import OrderedDict
from fdi.dataset.readonlydict import ReadOnlyDict
import copy
import logging
# create logger
logger = logging.getLogger(__name__)
# Data Model specification for mandatory components
_Model_Spec = {
'name': 'ArrayDataset_DataModel',
'description': 'ArrayDataset class data model mandatory configuration',
'parents': [
None,
],
'schema': '1.6',
'metadata': {
'description': {
'data_type': 'string',
'description': 'Description of this dataset',
'default': 'UNKNOWN',
'valid': '',
},
'type': {
'data_type': 'string',
'description': 'Type identification. Name of class.',
'default': 'ArrayDataset',
'valid': '',
},
'unit': {
'data_type': 'string',
'description': 'Unit of every element.',
'default': None,
'valid': '',
},
'typecode': {
'data_type': 'string',
'description': 'Python internal storage code.',
'default': 'UNKNOWN',
'valid': '',
},
'version': {
'data_type': 'string',
'description': 'Version of dataset',
'default': '0.1',
'valid': '',
},
'FORMATV': {
'data_type': 'string',
'description': 'Version of dataset schema and revision',
'default': '1.6.0.1',
'valid': '',
},
},
'datasets': {
},
}
Model = ReadOnlyDict(_Model_Spec)
# -*- coding: utf-8 -*-
from collections import ChainMap
import os
from .metadataholder import MetaDataHolder
from .metadata import AbstractParameter, Parameter
from .datatypes import DataTypes
from .deserialize import Class_Look_Up
import os
from collections import ChainMap
import logging
# create logger
logger = logging.getLogger(__name__)
#logger.debug('level %d' % (logger.getEffectiveLevel()))
# logger.debug('level %d' % (logger.getEffectiveLevel()))
MdpInfo = {}
def addMetaDataProperty(cls):
"""mh: Add MDP to a class so that although they are metadata,
they can be accessed by for example, productfoo.creator.
dynamic properties see
https://stackoverflow.com/a/2584050
https://stackoverflow.com/a/1355444
"""
# MdpInfo is evaluated at class import time
for name in MdpInfo:
def g(self, n=name):
return self._meta[name].getValue()
def s(self, value, n=name):
self.setMdp(n, value, MdpInfo)
def d(self, n=name):
logger.warn('Cannot delete MetaData Property ' + n)
setattr(cls, name, property(
g, s, d, 'MetaData Property ' + name))
return cls
# These property names are not for mormal properties
Reserved_Property_Names = ['history', 'meta',
'refs', 'dataset', 'zInfo', '_MDP', 'extraMdp']
def make_class_properties(attrs, reserved=None):
""" Generates class properties source code string according to given attribute names.
"""
pr = []
if reserved is None:
reserved = Reserved_Property_Names
for x in attrs:
if x in reserved:
raise NameError('"%s" is a reserved property name.' % x)
arg = x # x + '_' if x == 'type' else x
pr.append(' @property')
pr.append(' def %s(self): pass' % x)
# pr.append(' return self._meta["%s"].getValue()\n' % x)
# pr.append(' @%s.setter' % x)
# pr.append(' def %s(self, p):' % x)
# pr.append(' self.setMdp("%s", p, self._MDP)\n' % x)
pr.append(' pass')
properties = '\n'.join(pr)
return properties
class MetaDataProperties():
""" Mix-in place-holder for Class denifitions that has not their own `property.py` generated by :meth: `makeMdpMethods`.
"""
pass
Reserved_Property_Names = ['history', 'meta', 'refs', 'dataset',
'zInfo', '_MDP', 'extraMdp', 'alwaysMeta']
class Attributable(MetaDataHolder):
......@@ -88,19 +38,27 @@ class Attributable(MetaDataHolder):
self.alwaysMeta = alwaysMeta
self.extraMdp = {}
self.zInfo = zInfo if zInfo else {'metadata': {}}
# the list of arg names of the 'and' set
# andset = set(MdpInfo) & set(kwds)
if not alwaysMeta:
mdps = dict((x, kwds.pop('typ_' if x == 'type' else x))
for x in self.zInfo['metadata'])
self._MDP = ChainMap(self.extraMdp, self.zInfo['metadata'])
# super set of args and Model. Values only.
# if an MDP is set to None in args, use it Model default
mdps = {}
zm = self.zInfo['metadata']
for x in zm:
# 'type' in args is 'typ_'. put `type` back.
_x = 'typ_' if x == 'type' else x
if _x not in kwds:
mdps[x] = zm[x]['default']
else:
kwval = kwds.pop(_x)
mdps[x] = zm[x]['default'] if kwval is None else kwval
self._MDP = ChainMap(self.extraMdp, zm)
super(Attributable, self).__init__(meta=meta, **kwds)
self.setParameters(mdps)
def setParameters(self, params):
""" Set a group of name-value pairs to the object as properties.
params: a dictionary of name:value where value is a subclass of
params: a dictionary of name:value where value is a subclass of
`AbstractParameter`. value can be the value of a registered MDP.
``type`` will be used if ``typ_`` is given as the name.
"""
......@@ -121,7 +79,7 @@ class Attributable(MetaDataHolder):
self.setMeta(newMetadata)
def setMeta(self, newMetadata):
""" Replaces the current MetaData with specified argument.
""" Replaces the current MetaData with specified argument.
mh: Product will override this to add listener when meta is
replaced
......@@ -129,7 +87,7 @@ class Attributable(MetaDataHolder):
self._meta = newMetadata
def __getattribute__(self, name):
""" Returns the named metadata parameter.
""" Returns the named metadata parameter.
Reads meta data table when Attributes are
read, and returns the values only.
......@@ -150,13 +108,14 @@ class Attributable(MetaDataHolder):
def setMdp(self, name, value, met=None):
m = self.getMeta()
#print('MDP ', name, value, id(m), len(m))
# print('MDP ', name, value, id(m), len(m))
if name in m:
# meta already has a Parameter for name
p = m[name]
if issubclass(value.__class__, AbstractParameter):
tv, tp = value.getType(), p.getType()
if issubclass(tv, tp):
if issubclass(Class_Look_Up[DataTypes[tv]],
Class_Look_Up[DataTypes[tp]]):
p = value
return
else:
......@@ -166,8 +125,8 @@ class Attributable(MetaDataHolder):
else:
# value is not a Parameter
v_type = type(value)
p_type = type(p.value)
if issubclass(v_type, p_type):
p_type = Class_Look_Up[DataTypes[p.getType()]]
if value is None or issubclass(v_type, p_type):
p.setValue(value)
return
else:
......@@ -188,11 +147,11 @@ class Attributable(MetaDataHolder):
def __setattr__(self, name, value):
""" Stores value to attribute with name given.
If name is in the `zInfo` list, store the value in a Parameter in
metadata container. Updates meta data table. Updates value when
If name is in the `zInfo` list, store the value in a Parameter in
metadata container. Updates meta data table. Updates value when
an MDP attribute already has its Parameter in metadata.
value: Must be Parameter/NumericParameter if this is normal metadata,
value: Must be Parameter/NumericParameter if this is normal metadata,
depending on if it is `Number`. `Value` is the value if the attribute
is an MDP
......@@ -206,11 +165,11 @@ class Attributable(MetaDataHolder):
if self.alwaysMeta:
if issubclass(value.__class__, AbstractParameter):
# taken as an MDP attribute . store in meta
self.extraMdp[name] = None
self.extraMdp[name] = value
self.setMdp(name, value, self._MDP)
# must return without updating self.__dict__
return
elif name in self._MDP:
if name in self._MDP:
# an MDP attribute like 'description'. store in meta
self.setMdp(name, value, self._MDP)
# must return without updating self.__dict__
......@@ -307,3 +266,55 @@ def value2parameter(name, value, descriptor):
valid=gs,
)
return ret
def addMetaDataProperty(cls):
"""mh: Add MDP to a class so that although they are metadata,
they can be accessed by for example, productfoo.creator.
dynamic properties see
https://stackoverflow.com/a/2584050
https://stackoverflow.com/a/1355444
"""
# MdpInfo is evaluated at class import time
for name in MdpInfo:
def g(self, n=name):
return self._meta[name].getValue()
def s(self, value, n=name):
self.setMdp(n, value, MdpInfo)
def d(self, n=name):
logger.warn('Cannot delete MetaData Property ' + n)
setattr(cls, name, property(
g, s, d, 'MetaData Property ' + name))
return cls
def make_class_properties(attrs, reserved=None):
""" Generates class properties source code string according to given attribute names.
"""
pr = []
if reserved is None:
reserved = Reserved_Property_Names
for x in attrs:
if x in reserved:
raise NameError('"%s" is a reserved property name.' % x)
arg = x # x + '_' if x == 'type' else x
pr.append(' @property')
pr.append(' def %s(self): pass' % x)
# pr.append(' return self._meta["%s"].getValue()\n' % x)
# pr.append(' @%s.setter' % x)
# pr.append(' def %s(self, p):' % x)
# pr.append(' self.setMdp("%s", p, self._MDP)\n' % x)
pr.append(' pass')
properties = '\n'.join(pr)
return properties
class MetaDataProperties():
""" Mix-in place-holder for Class denifitions that has not their own `property.py` generated by :meth: `makeMdpMethods`.
"""
pass
......@@ -14,9 +14,8 @@ from fdi.dataset.eq import deepcmp
from fdi.dataset.copyable import Copyable
from fdi.dataset.history import History
import copy
from functools import lru_cache
from collections import OrderedDict
import itertools
import logging
# create logger
......@@ -24,12 +23,11 @@ logger = logging.getLogger(__name__)
class BaseProduct( AbstractComposite, Copyable, Serializable, EventSender):
""" A BaseProduct is a generic result that can be passed on between
(standalone) processes.
""" A BaseProduct is the starting point of te whole product tree, and a generic result that can be passed on between processes.
In general a Product contains zero or more datasets, history,
optional metadata as well as some required metadata fields.
Its intent is that it can fully describe itself; this includes
optional reference pointers and metadata some required metadata fields.
A Product is expected to fully describe itself; this includes
the way how this product was achieved (its history). As it is
the result of a process, it should be able to save to and restore
from an Archive device.
......@@ -39,42 +37,47 @@ class BaseProduct( AbstractComposite, Copyable, Serializable, EventSender):
method. Note that the datasets may be a composite of datasets
by themselves.
mh: Built-in Attributes in productInfo['metadata'] can be accessed with e.g. p.creator
or p.meta['description'].value:
p.creator='foo'
assert p.creatur=='foo'
assert p.meta['creator']=='foo'
p.meta['creator']=Parameter('bar')
assert p.meta['creator']==Parameter('bar')
A built-in attributes in `Model['metadata']` ("MetaData Parameter" or `MDP`) can be accessed with e.g. ``p.creator``, or p.meta['creator'].value::
BaseProduct class (level ALL) schema 1.5 inheriting [None].
p.creator='foo'
assert p.creatur=='foo'
assert p.meta['creator']=='foo'
p.meta['creator']=Parameter('bar')
assert p.meta['creator']==Parameter('bar')
Automatically generated from fdi/dataset/resources/BaseProduct.yml on 2021-06-03 22:39:08.610906.
=====
BaseProduct class schema 1.6 inheriting [None].
Automatically generated from fdi/dataset/resources/BaseProduct.yml on 2021-06-07 19:06:43.742425.
Description:
FDI base class
FDI base class data model
"""
def __init__(self,
description = 'UNKNOWN',
typ_ = 'BaseProduct',
level = 'ALL',
creator = 'UNKNOWN',
creationDate = FineTime(0),
rootCause = 'UNKNOWN',
version = '0.8',
FORMATV = '1.5.0.9',
FORMATV = '1.6.0.10',
zInfo=None,
**kwds):
# collect MDPs from args-turned-local-variables.
metasToBeInstalled = copy.copy(locals())
for x in ('self', '__class__', 'zInfo', 'kwds'):
metasToBeInstalled.pop(x)
global ProductInfo
metasToBeInstalled = OrderedDict(
itertools.filterfalse(
lambda x: x[0] in ('self', '__class__', 'zInfo', 'kwds'),
locals().items())
)
global Model
# instance variable for Model to be passed down inhritance chains.
if zInfo is None:
zInfo = ProductInfo
zInfo = Model
# must be the first line to initiate meta and zInfo
# :class: `Attributable` will process MDPs
......@@ -154,6 +157,8 @@ FDI base class
@property
def type(self): pass
@property
def level(self): pass
@property
def creator(self): pass
@property
def creationDate(self): pass
......@@ -165,14 +170,14 @@ FDI base class
def FORMATV(self): pass
pass
# Data Model specification for mandatory components
_Model_Spec = {
'name': 'BaseProduct',
'description': 'FDI base class',
'description': 'FDI base class data model',
'parents': [
None,
],
'level': 'ALL',
'schema': '1.5',
'schema': '1.6',
'metadata': {
'description': {
'id_zh_cn': '描述',
......@@ -192,6 +197,15 @@ _Model_Spec = {
'valid': '',
'typecode': 'B',
},
'level': {
'id_zh_cn': '产品xx',
'data_type': 'string',
'description': 'Product level.',
'description_zh_cn': '产品xx',
'default': 'ALL',
'valid': '',
'typecode': 'B',
},