diff --git a/ffflash/inc/nodelist.py b/ffflash/inc/nodelist.py index 6a1187a..8cdc83e 100644 --- a/ffflash/inc/nodelist.py +++ b/ffflash/inc/nodelist.py @@ -25,15 +25,13 @@ def _nodelist_fetch(ff): ) if not nodelist or not isinstance(nodelist, dict): return ff.log( - 'Could not fetch nodelist {}'.format(ff.args.nodelist), + 'could not fetch nodelist {}'.format(ff.args.nodelist), level=False ) - if not all([ - nodelist.get(a) for a in ['version', 'nodes', 'updated_at'] - ]): + if not all([(a in nodelist) for a in ['version', 'nodes', 'updated_at']]): return ff.log( - 'This is no nodelist {}'.format(ff.args.nodelist), + 'this is no nodelist {}'.format(ff.args.nodelist), level=False ) diff --git a/ffflash/inc/rankfile.py b/ffflash/inc/rankfile.py index 3992a7c..a8c13c3 100644 --- a/ffflash/inc/rankfile.py +++ b/ffflash/inc/rankfile.py @@ -1,3 +1,113 @@ +from operator import itemgetter +from os import path + +from ffflash.lib.api import api_timestamp +from ffflash.lib.files import check_file_location, dump_file, load_file + + +def _rankfile_load(ff): + if not ff.access_for('rankfile'): + return (False, None, None) + ff.log('handling rankfile {}'.format(ff.args.rankfile)) + + rankfile = check_file_location(ff.args.rankfile, must_exist=False) + if not rankfile: + return ff.log( + 'wrong path for rankfile {}'.format(ff.args.rankfile), + level=False + ), None, None + _, ext = path.splitext(rankfile) + if not ext or ext.lower() not in ['.yaml', '.json']: + return ff.log( + 'rankfile {} {} is neither json nor yaml'.format(rankfile, ext), + level=False + ), None, None + as_yaml = True if ext == '.yaml' else False + + ranks = load_file(rankfile, fallback={ + 'updated_at': 'never', 'nodes': [] + }, as_yaml=as_yaml) + + if not ranks or not isinstance(ranks, dict): + return ff.log( + 'could not load rankfile {}'.format(rankfile), + level=False + ), None, None + + if not all([(a in ranks) for a in ['nodes', 'updated_at']]): + return ff.log( + 'this is no rankfile {}'.format(rankfile), + level=False + ), None, None + + lranks = len(ranks.get('nodes', 0)) + ff.log(( + 'creating new rankfile {}'.format(rankfile) + if lranks == 0 else + 'loaded {} nodes'.format(lranks) + )) + + return rankfile, ranks, as_yaml + + +def _rankfile_score(ff, ranks, nodelist): + if not ff.access_for('rankfile'): + return False + if not all([ + ranks, isinstance(ranks, dict), ranks and 'nodes' in ranks, + nodelist, isinstance(nodelist, dict), nodelist and 'nodes' in nodelist, + ]): + return ff.log('wrong input data passed', level=False) + + res = [] + exist = dict( + (node.get('id'), node.get('score')) for node in ranks.get('nodes', []) + ) + for node in nodelist.get('nodes', []): + nid = node.get('id') + if not nid: + ff.log('node without id {}'.format(node)) + continue + nr = { + 'id': nid, + 'score': exist.get(nid, ff.args.rankwelcome), + 'name': node.get('name') + } + + if node.get('position', False): + nr['score'] += ff.args.rankposition + if node.get('status', {}).get('online', False): + nr['score'] += ff.args.rankonline + nr['online'] = True + cl = node.get('status', {}).get('clients', 0) + nr['score'] += (ff.args.rankclients * cl) + nr['clients'] = cl + else: + nr['score'] -= ff.args.rankoffline + nr['online'] = False + nr['clients'] = 0 + if nr['score'] > 0: + res.append(nr) + + ranks['nodes'] = list(sorted(res, key=itemgetter('score'), reverse=True)) + return ranks + + +def _rankfile_dump(ff, rankfile, ranks, as_yaml): + if not ff.access_for('rankfile'): + return False + if not all([ + rankfile, ranks, isinstance(ranks, dict), all([ + (a in ranks) for a in ['nodes', 'updated_at'] if ranks + ]), (as_yaml is not None) + ]): + return ff.log('wrong input data passed', level=False) + + ranks['updated_at'] = api_timestamp() + dump_file(rankfile, ranks, as_yaml=as_yaml) + + return True + def handle_rankfile(ff, nodelist): ''' @@ -11,3 +121,13 @@ def handle_rankfile(ff, nodelist): return False if not nodelist or not isinstance(nodelist, dict): return False + + rankfile, ranks, as_yaml = _rankfile_load(ff) + if not all([rankfile, ranks, (as_yaml is not None)]): + return False + + ranks = _rankfile_score(ff, ranks, nodelist) + if not ranks: + return False + + return _rankfile_dump(ff, rankfile, ranks, as_yaml) diff --git a/ffflash/info.py b/ffflash/info.py index 5d7c776..a1e04c4 100644 --- a/ffflash/info.py +++ b/ffflash/info.py @@ -19,7 +19,7 @@ def __init__(self): self.version = '0.9' self.name = self.cname.lower() - self.release = '{}{}'.format(self.version, 'a5') + self.release = '{}{}'.format(self.version, 'a6') self.ident = '{} {}'.format(self.name, self.release) self.download_url = '{}/archive/{}.tar.gz'.format( self.url, self.release diff --git a/ffflash/lib/args.py b/ffflash/lib/args.py index 53dabe5..7f4a897 100644 --- a/ffflash/lib/args.py +++ b/ffflash/lib/args.py @@ -21,6 +21,10 @@ def parsed_args(argv=None): 'APIfile', action='store', help='Freifunk API File to modify' ) + parser.add_argument( + '-s', '--sidecars', nargs='+', + help='sync updates from/with sidecar files' + ) parser.add_argument( '-n', '--nodelist', action='store', help='URL or location to map\'s nodelist.json, updates nodes count' @@ -30,8 +34,24 @@ def parsed_args(argv=None): help='location to rankfile.json, for node statistics and credits' ) parser.add_argument( - '-s', '--sidecars', nargs='+', - help='sync updates from/with sidecar files' + '-rc', '--rankclients', action='store', type=float, default=0.01, + help='factor to increase score per client' + ) + parser.add_argument( + '-rf', '--rankoffline', action='store', type=float, default=1.0, + help='score to decrease on offline' + ) + parser.add_argument( + '-rn', '--rankonline', action='store', type=float, default=1.0, + help='score to increase on online' + ) + parser.add_argument( + '-rp', '--rankposition', action='store', type=float, default=0.1, + help='score to increase on position set' + ) + parser.add_argument( + '-rw', '--rankwelcome', action='store', type=float, default=10.0, + help='score to start with for new nodes' ) parser.add_argument( '-d', '--dry', action='store_true', diff --git a/tests/inc/nodelist/test_handle_nodelist.py b/tests/inc/nodelist/test_handle_nodelist.py index 46dbe68..49ee1d3 100644 --- a/tests/inc/nodelist/test_handle_nodelist.py +++ b/tests/inc/nodelist/test_handle_nodelist.py @@ -1,4 +1,4 @@ -from json import dumps +from json import dumps, loads from random import choice from ffflash.inc.nodelist import handle_nodelist @@ -40,7 +40,7 @@ def test_handle_nodelist_empty_nodelist(tmpdir, fffake): assert handle_nodelist(ff) is False nl.write_text(dumps({ - 'version': 1, 'nodes': [{}], 'updated_at': 'never' + 'version': 1, 'nodes': [], 'updated_at': 'never' }), 'utf-8') ff = fffake(apifile, nodelist=nl, dry=True) assert handle_nodelist(ff) is False @@ -79,12 +79,19 @@ def test_handle_nodelist_launch_rankfile(tmpdir, fffake): apifile.write_text(dumps({'a': 'b'}), 'utf-8') nl = tmpdir.join('nodelist.json') nl.write_text(dumps({ - 'version': 1, 'nodes': [{}], 'updated_at': 'never' + 'version': 0, 'nodes': [], 'updated_at': 'never' }), 'utf-8') rf = tmpdir.join('rankfile.json') + assert tmpdir.listdir() == [apifile, nl] ff = fffake(apifile, nodelist=nl, rankfile=rf, dry=True) - assert handle_nodelist(ff) is False + assert handle_nodelist(ff) is True + assert tmpdir.listdir() == [apifile, nl, rf] + + res = loads(rf.read_text('utf-8')) + assert res + assert res.get('nodes') == [] + assert res.get('updated_at', False) != 'never' assert tmpdir.remove() is None diff --git a/tests/inc/nodelist/test_nodelist_fetch.py b/tests/inc/nodelist/test_nodelist_fetch.py index deab7c1..f9c2759 100644 --- a/tests/inc/nodelist/test_nodelist_fetch.py +++ b/tests/inc/nodelist/test_nodelist_fetch.py @@ -53,7 +53,7 @@ def test_nodelist_fetch_wrong_format_or_empty(tmpdir, fffake, capsys): def test_nodelist_fetch(tmpdir, fffake): apifile = tmpdir.join('api_file.json') apifile.write_text(dumps({'a': 'b'}), 'utf-8') - nodelist = {'version': 1, 'nodes': [None], 'updated_at': 'never'} + nodelist = {'version': 0, 'nodes': [], 'updated_at': 'never'} nl = tmpdir.join('nodelist.json') nl.write_text(dumps(nodelist), 'utf-8') assert tmpdir.listdir() == [apifile, nl] diff --git a/tests/inc/rankfile/test_handle_rankfile.py b/tests/inc/rankfile/test_handle_rankfile.py index f8034f3..b2dbae7 100644 --- a/tests/inc/rankfile/test_handle_rankfile.py +++ b/tests/inc/rankfile/test_handle_rankfile.py @@ -42,3 +42,39 @@ def test_handle_rankfile_empty_nodelist(tmpdir, fffake): assert handle_rankfile(ff, {}) is False assert tmpdir.remove() is None + + +def test_handle_rankfile_trashy_rankfile(tmpdir, fffake): + apifile = tmpdir.join('api_file.json') + apifile.write_text(dumps({'a': 'b'}), 'utf-8') + nodelist = tmpdir.join('nodelist.json') + nodelist.write_text(dumps({'a': 'b'}), 'utf-8') + rankfile = tmpdir.join('rankfile.json') + rankfile.write_text(dumps({'a': 'b'}), 'utf-8') + + ff = fffake( + apifile, nodelist=nodelist, + rankfile=rankfile, dry=True + ) + + assert handle_rankfile(ff, {'a': 'b'}) is False + + assert tmpdir.remove() is None + + +def test_handle_rankfile_trashy_nodelist(tmpdir, fffake): + apifile = tmpdir.join('api_file.json') + apifile.write_text(dumps({'a': 'b'}), 'utf-8') + nodelist = tmpdir.join('nodelist.json') + nodelist.write_text(dumps({'a': 'b'}), 'utf-8') + rankfile = tmpdir.join('rankfile.json') + rankfile.write_text(dumps({'nodes': [], 'updated_at': 'now'}), 'utf-8') + + ff = fffake( + apifile, nodelist=nodelist, + rankfile=rankfile, dry=True + ) + + assert handle_rankfile(ff, {'a': 'b'}) is False + + assert tmpdir.remove() is None diff --git a/tests/inc/rankfile/test_rankfile_dump.py b/tests/inc/rankfile/test_rankfile_dump.py new file mode 100644 index 0000000..bc17159 --- /dev/null +++ b/tests/inc/rankfile/test_rankfile_dump.py @@ -0,0 +1,53 @@ +from json import dumps, loads + +from ffflash.inc.rankfile import _rankfile_dump + + +def test_rankfile_dump_no_access(tmpdir, fffake): + ff = fffake(tmpdir.join('api_file.json'), dry=True) + + assert _rankfile_dump(ff, None, {}, {}) is False + + assert tmpdir.remove() is None + + +def test_rankfile_dump_wrong_input(tmpdir, fffake): + apifile = tmpdir.join('api_file.json') + apifile.write_text(dumps({'a': 'b'}), 'utf-8') + rf = tmpdir.join('rankfile.yaml') + + ff = fffake( + apifile, nodelist=tmpdir.join('nodelist.json'), + rankfile=tmpdir.join('rankfile.yaml'), dry=True + ) + + assert _rankfile_dump(ff, None, None, None) is False + assert _rankfile_dump(ff, 'test', {}, None) is False + assert _rankfile_dump(ff, str(rf), {}, True) is False + assert _rankfile_dump(ff, str(rf), {'nodes': []}, True) is False + assert _rankfile_dump(ff, str(rf), {'updated_at': 'now'}, True) is False + + assert tmpdir.remove() is None + + +def test_rankfile_dump_data(tmpdir, fffake): + apifile = tmpdir.join('api_file.json') + apifile.write_text(dumps({'a': 'b'}), 'utf-8') + rf = tmpdir.join('rankfile.yaml') + rk = {'updated_at': 'never', 'nodes': [{'a': 'b'}, {'c': 'd'}]} + + assert tmpdir.listdir() == [apifile] + ff = fffake( + apifile, nodelist=tmpdir.join('nodelist.json'), + rankfile=tmpdir.join('rankfile.json'), dry=True + ) + + assert _rankfile_dump(ff, str(rf), rk, False) is True + + assert tmpdir.listdir() == [apifile, rf] + r = loads(rf.read_text('utf-8')) + assert r + assert r.get('nodes') == rk['nodes'] + assert r.get('updated_at') != 'never' + + assert tmpdir.remove() is None diff --git a/tests/inc/rankfile/test_rankfile_load.py b/tests/inc/rankfile/test_rankfile_load.py new file mode 100644 index 0000000..4c8e812 --- /dev/null +++ b/tests/inc/rankfile/test_rankfile_load.py @@ -0,0 +1,120 @@ +from json import dumps + +from ffflash.inc.rankfile import _rankfile_load + + +def test_rankfile_load_no_access(tmpdir, fffake): + ff = fffake(tmpdir.join('api_file.json'), dry=True) + + assert _rankfile_load(ff) == (False, None, None) + + assert tmpdir.remove() is None + + +def test_rankfile_load_wrong_location(tmpdir, fffake): + apifile = tmpdir.join('api_file.json') + apifile.write_text(dumps({'a': 'b'}), 'utf-8') + rf = tmpdir.mkdir('rankfolder') + assert tmpdir.listdir() == [apifile, rf] + assert rf.listdir() == [] + + ff = fffake( + apifile, nodelist=tmpdir.join('nodelist.json'), rankfile=rf, dry=True + ) + assert ff.access_for('api') is True + assert ff.access_for('nodelist') is True + assert ff.access_for('rankfile') is True + + assert _rankfile_load(ff) == (False, None, None) + + assert tmpdir.remove() is None + + +def test_rankfile_load_wrong_extension(tmpdir, fffake, capsys): + apifile = tmpdir.join('api_file.json') + apifile.write_text(dumps({'a': 'b'}), 'utf-8') + + ff = fffake( + apifile, nodelist=tmpdir.join('nodelist.json'), + rankfile=tmpdir.join('rankfile.txt'), dry=True + ) + assert ff.access_for('api') is True + assert ff.access_for('nodelist') is True + assert ff.access_for('rankfile') is True + + assert _rankfile_load(ff) == (False, None, None) + out, err = capsys.readouterr() + assert 'ERROR' in out + assert 'json' in out + assert 'yaml' in out + assert err == '' + + assert tmpdir.remove() is None + + +def test_rankfile_load_non_existing_file(tmpdir, fffake): + apifile = tmpdir.join('api_file.json') + apifile.write_text(dumps({'a': 'b'}), 'utf-8') + rf = tmpdir.join('rankfile.yaml') + + ff = fffake( + apifile, nodelist=tmpdir.join('nodelist.json'), + rankfile=rf, dry=True + ) + + assert _rankfile_load(ff) == ( + str(rf), {'updated_at': 'never', 'nodes': []}, True + ) + + assert tmpdir.remove() is None + + +def test_rankfile_load_existing_file_with_errors(tmpdir, fffake, capsys): + apifile = tmpdir.join('api_file.json') + apifile.write_text(dumps({'a': 'b'}), 'utf-8') + + rf = tmpdir.join('rankfile.json') + rf.write_text(dumps(None), 'utf-8') + + ff = fffake( + apifile, nodelist=tmpdir.join('nodelist.json'), + rankfile=rf, dry=True + ) + assert _rankfile_load(ff) == (False, None, None) + out, err = capsys.readouterr() + assert 'ERROR' in out + assert 'could' in out + assert 'not' in out + assert err == '' + + rf = tmpdir.join('rankfile.json') + rf.write_text(dumps({'a': 'b'}), 'utf-8') + + ff = fffake( + apifile, nodelist=tmpdir.join('nodelist.json'), + rankfile=rf, dry=True + ) + assert _rankfile_load(ff) == (False, None, None) + out, err = capsys.readouterr() + assert 'ERROR' in out + assert 'is' in out + assert 'no' in out + assert err == '' + + assert tmpdir.remove() is None + + +def test_rankfile_load_existing_file(tmpdir, fffake): + apifile = tmpdir.join('api_file.json') + apifile.write_text(dumps({'a': 'b'}), 'utf-8') + rankfile = {'updated_at': 'now', 'nodes': [{'a': 'b'}, {'c': 'd'}]} + rf = tmpdir.join('rankfile.json') + rf.write_text(dumps(rankfile), 'utf-8') + + ff = fffake( + apifile, nodelist=tmpdir.join('nodelist.json'), + rankfile=rf, dry=True + ) + assert _rankfile_load(ff) == (str(rf), rankfile, False) + + assert tmpdir.remove() is None diff --git a/tests/inc/rankfile/test_rankfile_score.py b/tests/inc/rankfile/test_rankfile_score.py new file mode 100644 index 0000000..30dc316 --- /dev/null +++ b/tests/inc/rankfile/test_rankfile_score.py @@ -0,0 +1,132 @@ +from json import dumps + +from ffflash.inc.rankfile import _rankfile_score + + +def test_rankfile_score_no_access(tmpdir, fffake): + ff = fffake(tmpdir.join('api_file.json'), dry=True) + + assert _rankfile_score(ff, {}, {}) is False + + assert tmpdir.remove() is None + + +def test_rankfile_score_wrong_input(tmpdir, fffake): + apifile = tmpdir.join('api_file.json') + apifile.write_text(dumps({'a': 'b'}), 'utf-8') + ff = fffake( + apifile, nodelist=tmpdir.join('nodelist.json'), + rankfile=tmpdir.join('rankfile.json'), dry=True + ) + + assert _rankfile_score(ff, None, None) is False + assert _rankfile_score(ff, {}, {}) is False + assert _rankfile_score(ff, {'nodes': []}, {}) is False + assert _rankfile_score(ff, {}, {'nodes': []}) is False + + assert tmpdir.remove() is None + + +def test_rankfile_score_nodes_without_id(tmpdir, fffake, capsys): + apifile = tmpdir.join('api_file.json') + apifile.write_text(dumps({'a': 'b'}), 'utf-8') + ff = fffake( + apifile, nodelist=tmpdir.join('nodelist.json'), + rankfile=tmpdir.join('rankfile.json'), dry=True, verbose=True + ) + + assert _rankfile_score(ff, {'nodes': []}, {'nodes': []}) == {'nodes': []} + assert _rankfile_score(ff, {'nodes': []}, {'nodes': [{}]}) == {'nodes': []} + out, err = capsys.readouterr() + assert 'without' in out + assert 'id' in out + assert err == '' + + assert tmpdir.remove() is None + + +def test_rankfile_score_spot_known_nodes(tmpdir, fffake): + apifile = tmpdir.join('api_file.json') + apifile.write_text(dumps({'a': 'b'}), 'utf-8') + ff = fffake( + apifile, nodelist=tmpdir.join('nodelist.json'), + rankfile=tmpdir.join('rankfile.json'), dry=True, verbose=True + ) + + res = _rankfile_score(ff, {'nodes': [ + {'id': 'a', 'score': 23}, + {'id': 'b', 'score': 42}, + ]}, {'nodes': [ + {'id': 'a', 'name': 'A', 'status': {'online': True}}, + {'id': 'b', 'name': 'B'} + ]}) + assert res.get('nodes') + for n in res.get('nodes'): + assert n['clients'] == 0 + assert n['id'] == n['name'].lower() + if n['id'] == 'a': + assert n['online'] is True + assert n['score'] == (23 + ff.args.rankonline) + else: + assert n['online'] is False + assert n['score'] == (42 - ff.args.rankoffline) + + assert tmpdir.remove() is None + + +def test_rankfile_score_welcome_new_nodes(tmpdir, fffake): + apifile = tmpdir.join('api_file.json') + apifile.write_text(dumps({'a': 'b'}), 'utf-8') + ff = fffake( + apifile, nodelist=tmpdir.join('nodelist.json'), + rankfile=tmpdir.join('rankfile.json'), dry=True, verbose=True + ) + + res = _rankfile_score(ff, {'nodes': []}, {'nodes': [ + {'id': 'a', 'name': 'A', 'status': {'online': True}}, + {'id': 'b', 'name': 'B'} + ]}) + assert res.get('nodes') + for n in res.get('nodes'): + assert n['clients'] == 0 + assert n['id'] == n['name'].lower() + if n['id'] == 'a': + assert n['online'] is True + assert n['score'] == (ff.args.rankwelcome + ff.args.rankonline) + else: + assert n['online'] is False + assert n['score'] == (ff.args.rankwelcome - ff.args.rankoffline) + + assert tmpdir.remove() is None + + +def test_rankfile_score_rank_known_nodes(tmpdir, fffake): + apifile = tmpdir.join('api_file.json') + apifile.write_text(dumps({'a': 'b'}), 'utf-8') + ff = fffake( + apifile, nodelist=tmpdir.join('nodelist.json'), + rankfile=tmpdir.join('rankfile.json'), dry=True, verbose=True + ) + + res = _rankfile_score(ff, {'nodes': [ + {'id': 'a', 'score': 23}, + {'id': 'b', 'score': 42}, + ]}, {'nodes': [ + {'id': 'a', 'name': 'A', 'status': {'online': True, 'clients': 5}}, + {'id': 'b', 'name': 'B', 'position': {'lat': 0.0, 'lon': 0.0}} + ]}) + assert res.get('nodes') + for n in res.get('nodes'): + assert n['id'] == n['name'].lower() + if n['id'] == 'a': + assert n['online'] is True + assert n['score'] == ( + 23 + ff.args.rankonline + (5 * ff.args.rankclients) + ) + else: + assert n['online'] is False + assert n['score'] == ( + 42 - ff.args.rankoffline + ff.args.rankposition + ) + + assert tmpdir.remove() is None diff --git a/tests/lib/args/test_parsed_args.py b/tests/lib/args/test_parsed_args.py index 3c68a49..cab5ff1 100644 --- a/tests/lib/args/test_parsed_args.py +++ b/tests/lib/args/test_parsed_args.py @@ -49,6 +49,21 @@ def test_parsed_args_rankefile_with_or_without_nodelist(): assert parsed_args(ta) +def test_parsed_args_rank_details(): + a = vars(parsed_args([F])) + + for t in [ + 'rankclients', 'rankoffline', 'rankonline', + 'rankposition', 'rankwelcome' + ]: + sys_ex([F, '--{}'.format(t)]) + sys_ex([F, '--{}'.format(t), 'test']) + + v = a.get(t) + assert isinstance(v, float) + assert v > 0 + + def test_parsed_args_valid_options(): def t(a, nl=None, sc=None, rf=None, d=False, v=False): assert a.APIfile == F diff --git a/tests/main/test_run.py b/tests/main/test_run.py index 2123570..2f9735b 100644 --- a/tests/main/test_run.py +++ b/tests/main/test_run.py @@ -24,7 +24,7 @@ def test_run_dump_apifile(tmpdir, capsys): assert run([str(apifile), '-d']) is True out, err = capsys.readouterr() - assert out.strip().endswith(pformat(c)) + assert pformat(c) in out assert err == '' assert tmpdir.remove() is None