Alan/DCNmodel
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

copying to personal repo

Browse Source
This commit is contained in:
Alan
2022-06-19 13:45:53 -05:00
commit bf2ffa7315
287 changed files with 54032 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
cnmodel/util/difftreewidget/DataTreeWidget.py Normal file
View File

@@ -0,0 +1,131 @@
# -*- coding: utf-8 -*-
from pyqtgraph.Qt import QtGui, QtCore
from pyqtgraph.pgcollections import OrderedDict
from .TableWidget import TableWidget
from pyqtgraph.python2_3 import asUnicode
import types, traceback
import numpy as np
try:
import metaarray
HAVE_METAARRAY = True
except:
HAVE_METAARRAY = False
__all__ = ["DataTreeWidget"]
class DataTreeWidget(QtGui.QTreeWidget):
"""
Widget for displaying hierarchical python data structures
(eg, nested dicts, lists, and arrays)
"""
def __init__(self, parent=None, data=None):
QtGui.QTreeWidget.__init__(self, parent)
self.setVerticalScrollMode(self.ScrollPerPixel)
self.setData(data)
self.setColumnCount(3)
self.setHeaderLabels(["key / index", "type", "value"])
self.setAlternatingRowColors(True)
def setData(self, data, hideRoot=False):
"""data should be a dictionary."""
self.clear()
self.widgets = []
self.nodes = {}
self.buildTree(data, self.invisibleRootItem(), hideRoot=hideRoot)
self.expandToDepth(3)
self.resizeColumnToContents(0)
def buildTree(self, data, parent, name="", hideRoot=False, path=()):
if hideRoot:
node = parent
else:
node = QtGui.QTreeWidgetItem([name, "", ""])
parent.addChild(node)
# record the path to the node so it can be retrieved later
# (this is used by DiffTreeWidget)
self.nodes[path] = node
typeStr, desc, childs, widget = self.parse(data)
node.setText(1, typeStr)
node.setText(2, desc)
# Truncate description and add text box if needed
if len(desc) > 100:
desc = desc[:97] + "..."
if widget is None:
widget = QtGui.QPlainTextEdit(asUnicode(data))
widget.setMaximumHeight(200)
widget.setReadOnly(True)
# Add widget to new subnode
if widget is not None:
self.widgets.append(widget)
subnode = QtGui.QTreeWidgetItem(["", "", ""])
node.addChild(subnode)
self.setItemWidget(subnode, 0, widget)
self.setFirstItemColumnSpanned(subnode, True)
# recurse to children
for key, data in childs.items():
self.buildTree(data, node, asUnicode(key), path=path + (key,))
def parse(self, data):
"""
Given any python object, return:
* type
* a short string representation
* a dict of sub-objects to be parsed
* optional widget to display as sub-node
"""
# defaults for all objects
typeStr = type(data).__name__
if typeStr == "instance":
typeStr += ": " + data.__class__.__name__
widget = None
desc = ""
childs = {}
# type-specific changes
if isinstance(data, dict):
desc = "length=%d" % len(data)
if isinstance(data, OrderedDict):
childs = data
else:
childs = OrderedDict(sorted(data.items()))
elif isinstance(data, (list, tuple)):
desc = "length=%d" % len(data)
childs = OrderedDict(enumerate(data))
elif HAVE_METAARRAY and (
hasattr(data, "implements") and data.implements("MetaArray")
):
childs = OrderedDict(
[("data", data.view(np.ndarray)), ("meta", data.infoCopy())]
)
elif isinstance(data, np.ndarray):
desc = "shape=%s dtype=%s" % (data.shape, data.dtype)
table = TableWidget()
table.setData(data)
table.setMaximumHeight(200)
widget = table
elif isinstance(
data, types.TracebackType
): ## convert traceback to a list of strings
frames = list(
map(str.strip, traceback.format_list(traceback.extract_tb(data)))
)
# childs = OrderedDict([
# (i, {'file': child[0], 'line': child[1], 'function': child[2], 'code': child[3]})
# for i, child in enumerate(frames)])
# childs = OrderedDict([(i, ch) for i,ch in enumerate(frames)])
widget = QtGui.QPlainTextEdit(asUnicode("\n".join(frames)))
widget.setMaximumHeight(200)
widget.setReadOnly(True)
else:
desc = asUnicode(data)
return typeStr, desc, childs, widget

170
cnmodel/util/difftreewidget/DiffTreeWidget.py Normal file
View File

@@ -0,0 +1,170 @@
# -*- coding: utf-8 -*-
from pyqtgraph.Qt import QtGui, QtCore
from pyqtgraph.pgcollections import OrderedDict
from .DataTreeWidget import DataTreeWidget
import pyqtgraph.functions as fn
import types, traceback
import numpy as np
__all__ = ["DiffTreeWidget"]
class DiffTreeWidget(QtGui.QWidget):
"""
Widget for displaying differences between hierarchical python data structures
(eg, nested dicts, lists, and arrays)
"""
def __init__(self, parent=None, a=None, b=None):
QtGui.QWidget.__init__(self, parent)
self.layout = QtGui.QHBoxLayout()
self.setLayout(self.layout)
self.trees = [DataTreeWidget(self), DataTreeWidget(self)]
for t in self.trees:
self.layout.addWidget(t)
if a is not None:
self.setData(a, b)
def setData(self, a, b):
"""
Set the data to be compared in this widget.
"""
self.data = (a, b)
self.trees[0].setData(a)
self.trees[1].setData(b)
return self.compare(a, b)
def compare(self, a, b, path=()):
"""
Compare data structure *a* to structure *b*.
Return True if the objects match completely.
Otherwise, return a structure that describes the differences:
{ 'type': bool
'len': bool,
'str': bool,
'shape': bool,
'dtype': bool,
'mask': array,
}
"""
bad = (255, 200, 200)
diff = []
# generate typestr, desc, childs for each object
typeA, descA, childsA, _ = self.trees[0].parse(a)
typeB, descB, childsB, _ = self.trees[1].parse(b)
if typeA != typeB:
self.setColor(path, 1, bad)
if descA != descB:
self.setColor(path, 2, bad)
if isinstance(a, dict) and isinstance(b, dict):
keysA = set(a.keys())
keysB = set(b.keys())
for key in keysA - keysB:
self.setColor(path + (key,), 0, bad, tree=0)
for key in keysB - keysA:
self.setColor(path + (key,), 0, bad, tree=1)
for key in keysA & keysB:
self.compare(a[key], b[key], path + (key,))
elif isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)):
for i in range(max(len(a), len(b))):
if len(a) <= i:
self.setColor(path + (i,), 0, bad, tree=1)
elif len(b) <= i:
self.setColor(path + (i,), 0, bad, tree=0)
else:
self.compare(a[i], b[i], path + (i,))
elif (
isinstance(a, np.ndarray)
and isinstance(b, np.ndarray)
and a.shape == b.shape
):
tableNodes = [tree.nodes[path].child(0) for tree in self.trees]
if a.dtype.fields is None and b.dtype.fields is None:
eq = self.compareArrays(a, b)
if not np.all(eq):
for n in tableNodes:
n.setBackground(0, fn.mkBrush(bad))
# for i in np.argwhere(~eq):
else:
if a.dtype == b.dtype:
for i, k in enumerate(a.dtype.fields.keys()):
eq = self.compareArrays(a[k], b[k])
if not np.all(eq):
for n in tableNodes:
n.setBackground(0, fn.mkBrush(bad))
# for j in np.argwhere(~eq):
# dict: compare keys, then values where keys match
# list:
# array: compare elementwise for same shape
def compareArrays(self, a, b):
intnan = -9223372036854775808 # happens when np.nan is cast to int
anans = np.isnan(a) | (a == intnan)
bnans = np.isnan(b) | (b == intnan)
eq = anans == bnans
mask = ~anans
eq[mask] = np.allclose(a[mask], b[mask])
return eq
def setColor(self, path, column, color, tree=None):
brush = fn.mkBrush(color)
# Color only one tree if specified.
if tree is None:
trees = self.trees
else:
trees = [self.trees[tree]]
for tree in trees:
item = tree.nodes[path]
item.setBackground(column, brush)
def _compare(self, a, b):
"""
Compare data structure *a* to structure *b*.
"""
# Check test structures are the same
assert type(info) is type(expect)
if hasattr(info, "__len__"):
assert len(info) == len(expect)
if isinstance(info, dict):
for k in info:
assert k in expect
for k in expect:
assert k in info
self.compare_results(info[k], expect[k])
elif isinstance(info, list):
for i in range(len(info)):
self.compare_results(info[i], expect[i])
elif isinstance(info, np.ndarray):
assert info.shape == expect.shape
assert info.dtype == expect.dtype
if info.dtype.fields is None:
intnan = -9223372036854775808 # happens when np.nan is cast to int
inans = np.isnan(info) | (info == intnan)
enans = np.isnan(expect) | (expect == intnan)
assert np.all(inans == enans)
mask = ~inans
assert np.allclose(info[mask], expect[mask])
else:
for k in info.dtype.fields.keys():
self.compare_results(info[k], expect[k])
else:
try:
assert info == expect
except Exception:
raise NotImplementedError(
"Cannot compare objects of type %s" % type(info)
)

2
cnmodel/util/difftreewidget/README Normal file
View File

@@ -0,0 +1,2 @@
Copied from github.com/campagnola/pyqtgraph datatree-arrays branch; this gives us DiffTreeWidget.
This file can be removed if DiffTreeWidget has been merged into pyqtgraph.

515
cnmodel/util/difftreewidget/TableWidget.py Normal file
View File

@@ -0,0 +1,515 @@
# -*- coding: utf-8 -*-
import numpy as np
from pyqtgraph.Qt import QtGui, QtCore
from pyqtgraph.python2_3 import asUnicode, basestring
import pyqtgraph.metaarray as metaarray
__all__ = ["TableWidget"]
def _defersort(fn):
def defersort(self, *args, **kwds):
# may be called recursively; only the first call needs to block sorting
setSorting = False
if self._sorting is None:
self._sorting = self.isSortingEnabled()
setSorting = True
self.setSortingEnabled(False)
try:
return fn(self, *args, **kwds)
finally:
if setSorting:
self.setSortingEnabled(self._sorting)
self._sorting = None
return defersort
class TableWidget(QtGui.QTableWidget):
"""Extends QTableWidget with some useful functions for automatic data handling
and copy / export context menu. Can automatically format and display a variety
of data types (see :func:`setData() <pyqtgraph.TableWidget.setData>` for more
information.
"""
def __init__(self, *args, **kwds):
"""
All positional arguments are passed to QTableWidget.__init__().
===================== =================================================
**Keyword Arguments**
editable (bool) If True, cells in the table can be edited
by the user. Default is False.
sortable (bool) If True, the table may be soted by
clicking on column headers. Note that this also
causes rows to appear initially shuffled until
a sort column is selected. Default is True.
*(added in version 0.9.9)*
===================== =================================================
"""
QtGui.QTableWidget.__init__(self, *args)
self.itemClass = TableWidgetItem
self.setVerticalScrollMode(self.ScrollPerPixel)
self.setSelectionMode(QtGui.QAbstractItemView.ContiguousSelection)
self.setSizePolicy(QtGui.QSizePolicy.Preferred, QtGui.QSizePolicy.Preferred)
self.clear()
kwds.setdefault("sortable", True)
kwds.setdefault("editable", False)
self.setEditable(kwds.pop("editable"))
self.setSortingEnabled(kwds.pop("sortable"))
if len(kwds) > 0:
raise TypeError("Invalid keyword arguments '%s'" % kwds.keys())
self._sorting = None # used when temporarily disabling sorting
self._formats = {
None: None
} # stores per-column formats and entire table format
self.sortModes = {} # stores per-column sort mode
self.itemChanged.connect(self.handleItemChanged)
self.contextMenu = QtGui.QMenu()
self.contextMenu.addAction("Copy Selection").triggered.connect(self.copySel)
self.contextMenu.addAction("Copy All").triggered.connect(self.copyAll)
self.contextMenu.addAction("Save Selection").triggered.connect(self.saveSel)
self.contextMenu.addAction("Save All").triggered.connect(self.saveAll)
def clear(self):
"""Clear all contents from the table."""
QtGui.QTableWidget.clear(self)
self.verticalHeadersSet = False
self.horizontalHeadersSet = False
self.items = []
self.setRowCount(0)
self.setColumnCount(0)
self.sortModes = {}
def setData(self, data):
"""Set the data displayed in the table.
Allowed formats are:
* numpy arrays
* numpy record arrays
* metaarrays
* list-of-lists [[1,2,3], [4,5,6]]
* dict-of-lists {'x': [1,2,3], 'y': [4,5,6]}
* list-of-dicts [{'x': 1, 'y': 4}, {'x': 2, 'y': 5}, ...]
"""
self.clear()
self.appendData(data)
self.resizeColumnsToContents()
@_defersort
def appendData(self, data):
"""
Add new rows to the table.
See :func:`setData() <pyqtgraph.TableWidget.setData>` for accepted
data types.
"""
startRow = self.rowCount()
fn0, header0 = self.iteratorFn(data)
if fn0 is None:
self.clear()
return
it0 = fn0(data)
try:
first = next(it0)
except StopIteration:
return
fn1, header1 = self.iteratorFn(first)
if fn1 is None:
self.clear()
return
firstVals = [x for x in fn1(first)]
self.setColumnCount(len(firstVals))
if not self.verticalHeadersSet and header0 is not None:
labels = [self.verticalHeaderItem(i).text() for i in range(self.rowCount())]
self.setRowCount(startRow + len(header0))
self.setVerticalHeaderLabels(labels + header0)
self.verticalHeadersSet = True
if not self.horizontalHeadersSet and header1 is not None:
self.setHorizontalHeaderLabels(header1)
self.horizontalHeadersSet = True
i = startRow
self.setRow(i, firstVals)
for row in it0:
i += 1
self.setRow(i, [x for x in fn1(row)])
if (
self._sorting
and self.horizontalHeadersSet
and self.horizontalHeader().sortIndicatorSection() >= self.columnCount()
):
self.sortByColumn(0, QtCore.Qt.AscendingOrder)
def setEditable(self, editable=True):
self.editable = editable
for item in self.items:
item.setEditable(editable)
def setFormat(self, format, column=None):
"""
Specify the default text formatting for the entire table, or for a
single column if *column* is specified.
If a string is specified, it is used as a format string for converting
float values (and all other types are converted using str). If a
function is specified, it will be called with the item as its only
argument and must return a string. Setting format = None causes the
default formatter to be used instead.
Added in version 0.9.9.
"""
if (
format is not None
and not isinstance(format, basestring)
and not callable(format)
):
raise ValueError(
"Format argument must string, callable, or None. (got %s)" % format
)
self._formats[column] = format
if column is None:
# update format of all items that do not have a column format
# specified
for c in range(self.columnCount()):
if self._formats.get(c, None) is None:
for r in range(self.rowCount()):
item = self.item(r, c)
if item is None:
continue
item.setFormat(format)
else:
# set all items in the column to use this format, or the default
# table format if None was specified.
if format is None:
format = self._formats[None]
for r in range(self.rowCount()):
item = self.item(r, column)
if item is None:
continue
item.setFormat(format)
def iteratorFn(self, data):
## Return 1) a function that will provide an iterator for data and 2) a list of header strings
if isinstance(data, list) or isinstance(data, tuple):
return lambda d: d.__iter__(), None
elif isinstance(data, dict):
return lambda d: iter(d.values()), list(map(asUnicode, data.keys()))
elif hasattr(data, "implements") and data.implements("MetaArray"):
if data.axisHasColumns(0):
header = [
asUnicode(data.columnName(0, i)) for i in range(data.shape[0])
]
elif data.axisHasValues(0):
header = list(map(asUnicode, data.xvals(0)))
else:
header = None
return self.iterFirstAxis, header
elif isinstance(data, np.ndarray):
return self.iterFirstAxis, None
elif isinstance(data, np.void):
return self.iterate, list(map(asUnicode, data.dtype.names))
elif data is None:
return (None, None)
elif np.isscalar(data):
return self.iterateScalar, None
else:
msg = "Don't know how to iterate over data type: {!s}".format(type(data))
raise TypeError(msg)
def iterFirstAxis(self, data):
for i in range(data.shape[0]):
yield data[i]
def iterate(self, data):
# for numpy.void, which can be iterated but mysteriously
# has no __iter__ (??)
for x in data:
yield x
def iterateScalar(self, data):
yield data
def appendRow(self, data):
self.appendData([data])
@_defersort
def addRow(self, vals):
row = self.rowCount()
self.setRowCount(row + 1)
self.setRow(row, vals)
@_defersort
def setRow(self, row, vals):
if row > self.rowCount() - 1:
self.setRowCount(row + 1)
for col in range(len(vals)):
val = vals[col]
item = self.itemClass(val, row)
item.setEditable(self.editable)
sortMode = self.sortModes.get(col, None)
if sortMode is not None:
item.setSortMode(sortMode)
format = self._formats.get(col, self._formats[None])
item.setFormat(format)
self.items.append(item)
self.setItem(row, col, item)
item.setValue(val) # Required--the text-change callback is invoked
# when we call setItem.
def setSortMode(self, column, mode):
"""
Set the mode used to sort *column*.
============== ========================================================
**Sort Modes**
value Compares item.value if available; falls back to text
comparison.
text Compares item.text()
index Compares by the order in which items were inserted.
============== ========================================================
Added in version 0.9.9
"""
for r in range(self.rowCount()):
item = self.item(r, column)
if hasattr(item, "setSortMode"):
item.setSortMode(mode)
self.sortModes[column] = mode
def sizeHint(self):
# based on http://stackoverflow.com/a/7195443/54056
width = sum(self.columnWidth(i) for i in range(self.columnCount()))
width += self.verticalHeader().sizeHint().width()
width += self.verticalScrollBar().sizeHint().width()
width += self.frameWidth() * 2
height = sum(self.rowHeight(i) for i in range(self.rowCount()))
height += self.verticalHeader().sizeHint().height()
height += self.horizontalScrollBar().sizeHint().height()
return QtCore.QSize(width, height)
def serialize(self, useSelection=False):
"""Convert entire table (or just selected area) into tab-separated text values"""
if useSelection:
selection = self.selectedRanges()[0]
rows = list(range(selection.topRow(), selection.bottomRow() + 1))
columns = list(range(selection.leftColumn(), selection.rightColumn() + 1))
else:
rows = list(range(self.rowCount()))
columns = list(range(self.columnCount()))
data = []
if self.horizontalHeadersSet:
row = []
if self.verticalHeadersSet:
row.append(asUnicode(""))
for c in columns:
row.append(asUnicode(self.horizontalHeaderItem(c).text()))
data.append(row)
for r in rows:
row = []
if self.verticalHeadersSet:
row.append(asUnicode(self.verticalHeaderItem(r).text()))
for c in columns:
item = self.item(r, c)
if item is not None:
row.append(asUnicode(item.value))
else:
row.append(asUnicode(""))
data.append(row)
s = ""
for row in data:
s += "\t".join(row) + "\n"
return s
def copySel(self):
"""Copy selected data to clipboard."""
QtGui.QApplication.clipboard().setText(self.serialize(useSelection=True))
def copyAll(self):
"""Copy all data to clipboard."""
QtGui.QApplication.clipboard().setText(self.serialize(useSelection=False))
def saveSel(self):
"""Save selected data to file."""
self.save(self.serialize(useSelection=True))
def saveAll(self):
"""Save all data to file."""
self.save(self.serialize(useSelection=False))
def save(self, data):
fileName = QtGui.QFileDialog.getSaveFileName(
self, "Save As..", "", "Tab-separated values (*.tsv)"
)
if fileName == "":
return
open(fileName, "w").write(data)
def contextMenuEvent(self, ev):
self.contextMenu.popup(ev.globalPos())
def keyPressEvent(self, ev):
if ev.key() == QtCore.Qt.Key_C and ev.modifiers() == QtCore.Qt.ControlModifier:
ev.accept()
self.copySel()
else:
QtGui.QTableWidget.keyPressEvent(self, ev)
def handleItemChanged(self, item):
item.itemChanged()
class TableWidgetItem(QtGui.QTableWidgetItem):
def __init__(self, val, index, format=None):
QtGui.QTableWidgetItem.__init__(self, "")
self._blockValueChange = False
self._format = None
self._defaultFormat = "%0.3g"
self.sortMode = "value"
self.index = index
flags = QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled
self.setFlags(flags)
self.setValue(val)
self.setFormat(format)
def setEditable(self, editable):
"""
Set whether this item is user-editable.
"""
if editable:
self.setFlags(self.flags() | QtCore.Qt.ItemIsEditable)
else:
self.setFlags(self.flags() & ~QtCore.Qt.ItemIsEditable)
def setSortMode(self, mode):
"""
Set the mode used to sort this item against others in its column.
============== ========================================================
**Sort Modes**
value Compares item.value if available; falls back to text
comparison.
text Compares item.text()
index Compares by the order in which items were inserted.
============== ========================================================
"""
modes = ("value", "text", "index", None)
if mode not in modes:
raise ValueError("Sort mode must be one of %s" % str(modes))
self.sortMode = mode
def setFormat(self, fmt):
"""Define the conversion from item value to displayed text.
If a string is specified, it is used as a format string for converting
float values (and all other types are converted using str). If a
function is specified, it will be called with the item as its only
argument and must return a string.
Added in version 0.9.9.
"""
if fmt is not None and not isinstance(fmt, basestring) and not callable(fmt):
raise ValueError(
"Format argument must string, callable, or None. (got %s)" % fmt
)
self._format = fmt
self._updateText()
def _updateText(self):
self._blockValueChange = True
try:
self._text = self.format()
self.setText(self._text)
finally:
self._blockValueChange = False
def setValue(self, value):
self.value = value
self._updateText()
def itemChanged(self):
"""Called when the data of this item has changed."""
if self.text() != self._text:
self.textChanged()
def textChanged(self):
"""Called when this item's text has changed for any reason."""
self._text = self.text()
if self._blockValueChange:
# text change was result of value or format change; do not
# propagate.
return
try:
self.value = type(self.value)(self.text())
except ValueError:
self.value = str(self.text())
def format(self):
if callable(self._format):
return self._format(self)
if isinstance(self.value, (float, np.floating)):
if self._format is None:
return self._defaultFormat % self.value
else:
return self._format % self.value
else:
return asUnicode(self.value)
def __lt__(self, other):
if self.sortMode == "index" and hasattr(other, "index"):
return self.index < other.index
if self.sortMode == "value" and hasattr(other, "value"):
return self.value < other.value
else:
return self.text() < other.text()
if __name__ == "__main__":
app = QtGui.QApplication([])
win = QtGui.QMainWindow()
t = TableWidget()
win.setCentralWidget(t)
win.resize(800, 600)
win.show()
ll = [[1, 2, 3, 4, 5]] * 20
ld = [{"x": 1, "y": 2, "z": 3}] * 20
dl = {"x": list(range(20)), "y": list(range(20)), "z": list(range(20))}
a = np.ones((20, 5))
ra = np.ones((20,), dtype=[("x", int), ("y", int), ("z", int)])
t.setData(ll)
ma = metaarray.MetaArray(
np.ones((20, 3)),
info=[
{"values": np.linspace(1, 5, 20)},
{"cols": [{"name": "x"}, {"name": "y"}, {"name": "z"}]},
],
)
t.setData(ma)

1
cnmodel/util/difftreewidget/__init__.py Normal file
View File

@@ -0,0 +1 @@
from .DiffTreeWidget import DiffTreeWidget