diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d4df70a --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +hide +.dropbox +.DS_Store diff --git a/@double/tfocs_dot.m b/@double/tfocs_dot.m new file mode 100644 index 0000000..3503a7d --- /dev/null +++ b/@double/tfocs_dot.m @@ -0,0 +1,80 @@ +function v = tfocs_dot( x, y ) +% TFOCS_DOT Dot product . Returns real(x'*y) +% For matrices, this is the inner product that induces the Frobenius +% norm, i.e. = tr(x'*y), and not matrix multiplication. + +% Note: this is real(x'*y) and not real(x.'*y) + +if isempty( x ) || isempty( y ), + v = 0; + return; +end + +% Allow scalar times vector multiplies: +if isscalar(x) && ~isscalar(y) + if ~x, + v = 0; + return; + else +% x = repmat(x,size(y,1),size(y,2) ); % doesn't work if y is a cell + % The above code fails if y is multi-dimensional. Also, not + % memory efficient. Switching to this (10/9/2013) + % (Thanks to Graham Coleman for finding this bug, btw) + if issparse(y) + v = real( x * sum(nonzeros(y)) ); + else + v = real( x * sum(y(:)) ); + end + return; + end +elseif isscalar(y) && ~isscalar(x) + if ~y + v = 0; + return; + else + y = repmat(y,size(x,1),size(x,2) ); + if issparse(x) + v = real( y * sum(nonzeros(x)) ); + else + v = real( y * sum(x(:)) ); + end + return; + + end +end + +if isreal( x ) || isreal( y ), + if issparse( x ) || issparse( y ), + v = sum( nonzeros( real(x) .* real(y) ) ); + else + % Split this into two cases (first case could be handled by + % second case, but we're trying to make it very fast since + % this code is called very often) + if ndims(x)==2 && ndims(y)==2 && size(x,2) == 1 && size(y,2) == 1 && isreal(x) && isreal(y) + v = sum( x'*y ); % do we really need 'sum' ? + else + % Take real part first (since one of x and y is real anyhow) + % in order to save some computation: + v = real(x(:))' * real(y(:)); + end + end +else + if issparse( x ) || issparse( y ), + v = sum( nonzeros( real(x) .* real(y) ) ) + ... + sum( nonzeros( imag(x) .* imag(y) ) ); + else + % SRB: this is very slow: +% v = sum( real(x(:))' * real(y(:)) ) + ... +% sum( imag(x(:))' * imag(y(:)) ); + if ndims(x)==2 && ndims(y)==2 && size(x,2) == 1 && size(y,2) == 1 + v = sum(real( x'*y ) ); + else + % This is the most generic code. + v = real( x(:)'*y(:) ); + end + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. \ No newline at end of file diff --git a/@double/tfocs_size.m b/@double/tfocs_size.m new file mode 100644 index 0000000..ac0b208 --- /dev/null +++ b/@double/tfocs_size.m @@ -0,0 +1,9 @@ +function v = tfocs_size( x ) + +% SIZE TFOCS-friendly size operator. + +v = { @zeros, m, n }; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@double/vec.m b/@double/vec.m new file mode 100644 index 0000000..1e37e52 --- /dev/null +++ b/@double/vec.m @@ -0,0 +1,11 @@ +function v = vec( x ) + +% VEC Vectorize. + +v = reshape( x, numel(x), 1 ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + + \ No newline at end of file diff --git a/@single/tfocs_dot.m b/@single/tfocs_dot.m new file mode 100644 index 0000000..3503a7d --- /dev/null +++ b/@single/tfocs_dot.m @@ -0,0 +1,80 @@ +function v = tfocs_dot( x, y ) +% TFOCS_DOT Dot product . Returns real(x'*y) +% For matrices, this is the inner product that induces the Frobenius +% norm, i.e. = tr(x'*y), and not matrix multiplication. + +% Note: this is real(x'*y) and not real(x.'*y) + +if isempty( x ) || isempty( y ), + v = 0; + return; +end + +% Allow scalar times vector multiplies: +if isscalar(x) && ~isscalar(y) + if ~x, + v = 0; + return; + else +% x = repmat(x,size(y,1),size(y,2) ); % doesn't work if y is a cell + % The above code fails if y is multi-dimensional. Also, not + % memory efficient. Switching to this (10/9/2013) + % (Thanks to Graham Coleman for finding this bug, btw) + if issparse(y) + v = real( x * sum(nonzeros(y)) ); + else + v = real( x * sum(y(:)) ); + end + return; + end +elseif isscalar(y) && ~isscalar(x) + if ~y + v = 0; + return; + else + y = repmat(y,size(x,1),size(x,2) ); + if issparse(x) + v = real( y * sum(nonzeros(x)) ); + else + v = real( y * sum(x(:)) ); + end + return; + + end +end + +if isreal( x ) || isreal( y ), + if issparse( x ) || issparse( y ), + v = sum( nonzeros( real(x) .* real(y) ) ); + else + % Split this into two cases (first case could be handled by + % second case, but we're trying to make it very fast since + % this code is called very often) + if ndims(x)==2 && ndims(y)==2 && size(x,2) == 1 && size(y,2) == 1 && isreal(x) && isreal(y) + v = sum( x'*y ); % do we really need 'sum' ? + else + % Take real part first (since one of x and y is real anyhow) + % in order to save some computation: + v = real(x(:))' * real(y(:)); + end + end +else + if issparse( x ) || issparse( y ), + v = sum( nonzeros( real(x) .* real(y) ) ) + ... + sum( nonzeros( imag(x) .* imag(y) ) ); + else + % SRB: this is very slow: +% v = sum( real(x(:))' * real(y(:)) ) + ... +% sum( imag(x(:))' * imag(y(:)) ); + if ndims(x)==2 && ndims(y)==2 && size(x,2) == 1 && size(y,2) == 1 + v = sum(real( x'*y ) ); + else + % This is the most generic code. + v = real( x(:)'*y(:) ); + end + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. \ No newline at end of file diff --git a/@single/tfocs_size.m b/@single/tfocs_size.m new file mode 100644 index 0000000..ac0b208 --- /dev/null +++ b/@single/tfocs_size.m @@ -0,0 +1,9 @@ +function v = tfocs_size( x ) + +% SIZE TFOCS-friendly size operator. + +v = { @zeros, m, n }; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@single/vec.m b/@single/vec.m new file mode 100644 index 0000000..1e37e52 --- /dev/null +++ b/@single/vec.m @@ -0,0 +1,11 @@ +function v = vec( x ) + +% VEC Vectorize. + +v = reshape( x, numel(x), 1 ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + + \ No newline at end of file diff --git a/@tfocs_tuple/abs.m b/@tfocs_tuple/abs.m new file mode 100644 index 0000000..69568e9 --- /dev/null +++ b/@tfocs_tuple/abs.m @@ -0,0 +1,12 @@ +function x = abs( x ) + +% ABS Absolute value. + +n = numel( x.value_ ); +for k = 1 : n, + x.value_{k} = abs( x.value_{k} ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/cell.m b/@tfocs_tuple/cell.m new file mode 100644 index 0000000..ac7bb87 --- /dev/null +++ b/@tfocs_tuple/cell.m @@ -0,0 +1,9 @@ +function v = cell( x ) + +% CELL Conversion to a cell array. + +v = x.value_; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/disp.m b/@tfocs_tuple/disp.m new file mode 100644 index 0000000..d7066cc --- /dev/null +++ b/@tfocs_tuple/disp.m @@ -0,0 +1,37 @@ +function disp( x, prefix, inpname, nohead ) + +% DISP Manual display. DISP(x,prefix) adds the string prefix to each +% line of the display output. + +if nargin < 3, inpname = ''; end +if nargin < 2, prefix = ''; end +n = numel( x.value_ ); +if nargin < 4, + fprintf( '%stfocs tuple object:\n', prefix ); + prefix = [ prefix, ' ' ]; +end +for k = 1 : n, + ss = x.value_{k}; + inpname2 = sprintf( '%s{%d}', inpname, k ); + if isnumeric( ss ), + cls = class( ss ); + sz = size( ss ); + temp = sprintf( '%dx', sz ); + if all( sz == 1 ), + fprintf( '%s%s: [%g]\n', prefix, inpname2, ss ); + elseif isreal(ss), + fprintf( '%s%s: [%s %s]\n', prefix, inpname2, temp(1:end-1), cls ); + else + fprintf( '%s%s: [%s %s complex]\n', prefix, inpname2, temp(1:end-1), cls ); + end + elseif isa( ss, 'tfocs_tuple' ), + fprintf( '%s%s: tfocs tuple object\n', prefix, inpname2 ); + disp( ss, [ prefix, inpname2 ], '', 1 ); + else + fprintf( '%s%s: %s\n', prefix, inpname2, class(ss) ); + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/display.m b/@tfocs_tuple/display.m new file mode 100644 index 0000000..ff45db7 --- /dev/null +++ b/@tfocs_tuple/display.m @@ -0,0 +1,14 @@ +function display( x ) + +% DISPLAY Automatic display. + +long = ~isequal(get(0,'FormatSpacing'),'compact'); +if long, disp( ' ' ); end +disp([inputname(1) ' =']); +if long, disp( ' ' ); end +disp(x,' ',inputname(1)) +if long, disp( ' ' ); end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/get.m b/@tfocs_tuple/get.m new file mode 100644 index 0000000..05f95ca --- /dev/null +++ b/@tfocs_tuple/get.m @@ -0,0 +1,12 @@ +function y = get( x, ndxs ) +if nargin == 0, + y = x.value_; +elseif numel(ndxs) == 1, + y = x.value_{ndxs}; +else + y = x.value_(ndxs); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/max.m b/@tfocs_tuple/max.m new file mode 100644 index 0000000..66bdfc4 --- /dev/null +++ b/@tfocs_tuple/max.m @@ -0,0 +1,25 @@ +function z = max( x, y ) + +% MAX Maximum, z = max(x,y) + +if isa(x,'tfocs_tuple') + z = x; + if isa(y,'tfocs_tuple') + z.value_ = cellfun( @max, x.value_, y.value_, 'UniformOutput', false ); + elseif isscalar(y) + z.value_ = cellfun( @max, x.value_, {y}, 'UniformOutput', false ); + else + z.value_ = cellfun( @max, x.value_, {y}, 'UniformOutput', false ); + end +else + z = y; + if isscalar(x) + z.value_ = cellfun( @max, {x}, y.value_, 'UniformOutput', false ); + else + z.value_ = cellfun( @max, {x}, y.value_, 'UniformOutput', false ); + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/min.m b/@tfocs_tuple/min.m new file mode 100644 index 0000000..0c1f372 --- /dev/null +++ b/@tfocs_tuple/min.m @@ -0,0 +1,25 @@ +function z = min( x, y ) + +% MAX Minimum, z = min(x,y) + +if isa(x,'tfocs_tuple') + z = x; + if isa(y,'tfocs_tuple') + z.value_ = cellfun( @min, x.value_, y.value_, 'UniformOutput', false ); + elseif isscalar(y) + z.value_ = cellfun( @min, x.value_, {y}, 'UniformOutput', false ); + else + z.value_ = cellfun( @min, x.value_, {y}, 'UniformOutput', false ); + end +else + z = y; + if isscalar(x) + z.value_ = cellfun( @min, {x}, y.value_, 'UniformOutput', false ); + else + z.value_ = cellfun( @min, {x}, y.value_, 'UniformOutput', false ); + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/minus.m b/@tfocs_tuple/minus.m new file mode 100644 index 0000000..1c07f26 --- /dev/null +++ b/@tfocs_tuple/minus.m @@ -0,0 +1,13 @@ +function x = minus( x, y ) + +% MINUS Subtraction. + +if isnumeric( x ) && isscalar( x ) && x == 0, + x = -y; +elseif ~isnumeric( y ) || numel( y ) ~= 1 || y ~= 0 + x.value_ = cellfun( @minus, x.value_, y.value_, 'UniformOutput', false ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/mtimes.m b/@tfocs_tuple/mtimes.m new file mode 100644 index 0000000..b2a7ed5 --- /dev/null +++ b/@tfocs_tuple/mtimes.m @@ -0,0 +1,12 @@ +function y = mtimes( x, y ) + +% MTIMES Multiplication. TFOCS_TUPLE objects may only be left-multiplied +% by real scalars. + +for k = 1 : numel( y.value_ ), + y.value_{k} = x * y.value_{k}; +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/nnz.m b/@tfocs_tuple/nnz.m new file mode 100644 index 0000000..f029176 --- /dev/null +++ b/@tfocs_tuple/nnz.m @@ -0,0 +1,6 @@ +function ans = nnz( x ) +ans = sum( cellfun( @nnz, x.value_ ) ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/numel.m b/@tfocs_tuple/numel.m new file mode 100644 index 0000000..3cdc284 --- /dev/null +++ b/@tfocs_tuple/numel.m @@ -0,0 +1,9 @@ +function v = numel( x, varargin ) + +% NUMEL Number of elements. + +v = numel( x.value_, varargin{:} ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/plus.m b/@tfocs_tuple/plus.m new file mode 100644 index 0000000..5769de5 --- /dev/null +++ b/@tfocs_tuple/plus.m @@ -0,0 +1,13 @@ +function x = plus( x, y ) + +% PLUS Addition. + +if isnumeric( x ) && isscalar( x ) && x == 0, + x = y; +elseif ~isnumeric( y ) || numel( y ) ~= 1 || y ~= 0 + x.value_ = cellfun( @plus, x.value_, y.value_, 'UniformOutput', false ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/power.m b/@tfocs_tuple/power.m new file mode 100644 index 0000000..192c068 --- /dev/null +++ b/@tfocs_tuple/power.m @@ -0,0 +1,15 @@ +function x = power( x, y ) + +% POWER Matrix power, z = x.^y + +if isa(y,'tfocs_tuple') + x.value_ = cellfun( @power, x.value_, y.value_, 'UniformOutput', false ); +elseif isscalar(y) + x.value_ = cellfun( @power, x.value_, {y}, 'UniformOutput', false ); +else + x.value_ = cellfun( @power, x.value_, {y}, 'UniformOutput', false ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/size.m b/@tfocs_tuple/size.m new file mode 100644 index 0000000..8813a60 --- /dev/null +++ b/@tfocs_tuple/size.m @@ -0,0 +1,9 @@ +function v = size( x ) + +% SIZE Size. + +v = cellfun( @size, x.value_, 'UniformOutput', false ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/subsref.m b/@tfocs_tuple/subsref.m new file mode 100644 index 0000000..b75ec16 --- /dev/null +++ b/@tfocs_tuple/subsref.m @@ -0,0 +1,6 @@ +function varargout = subsref( x, varargin ) +[ varargout{1:nargout} ] = subsref( x.value_, varargin{:} ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/tfocs_dot.m b/@tfocs_tuple/tfocs_dot.m new file mode 100644 index 0000000..8d07290 --- /dev/null +++ b/@tfocs_tuple/tfocs_dot.m @@ -0,0 +1,13 @@ +function v = tfocs_dot( x, y ) + +% TFOCS_DOT Dot products. + +if isempty( x ) || isempty( y ), + v = 0; +else + v = sum( cellfun( @tfocs_dot, x.value_, y.value_ ) ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/tfocs_normsq.m b/@tfocs_tuple/tfocs_normsq.m new file mode 100644 index 0000000..d49e89e --- /dev/null +++ b/@tfocs_tuple/tfocs_normsq.m @@ -0,0 +1,12 @@ +function v = tfocs_normsq( x ) + +% TFOCS_NORMSQ Squared norm. By default, TFOCS_NORMSQ(X) is equal +% to TFOCS_NORMSQ(X,X), and this numerical equivalence +% must be preserved. However, an object may overload +% TFOCS_NORMSQ to compute its value more efficiently. + +v = sum( cellfun( @tfocs_normsq, x.value_ ) ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/tfocs_size.m b/@tfocs_tuple/tfocs_size.m new file mode 100644 index 0000000..999b056 --- /dev/null +++ b/@tfocs_tuple/tfocs_size.m @@ -0,0 +1,7 @@ +function v = tfocs_size( x ) + +v = { @tfocs_tuple, cellfun( @size, x.value_, 'UniformOutput', false ) }; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/tfocs_tuple.m b/@tfocs_tuple/tfocs_tuple.m new file mode 100644 index 0000000..bc30eef --- /dev/null +++ b/@tfocs_tuple/tfocs_tuple.m @@ -0,0 +1,24 @@ +function v = tfocs_tuple( w ) + +% TFOCS_TUPLE The TFOCS tuple object. +% This object is used to create tuples, which are elements of +% vector spaces that are Cartesian products of other vector +% spaces. TFOCS assumes that any element in a tfocs_tuple can +% perform the following basic operations: +% --- addition (plus) +% --- subtraction (minus) +% --- multiplication by real scalars (times,mtimes) +% --- dot products (tfocs_dot) +% --- squared norm (tfocs_normsq, optional) +% --- size (size; single-argument calls only) + +if ~iscell(w) && ~isempty(w) + error('tfocs_tuple constructor: input must be a cell array'); +end +v.value_ = reshape( w, 1, numel(w) ); +v = class( v, 'tfocs_tuple' ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/@tfocs_tuple/tfocs_zeros.m b/@tfocs_tuple/tfocs_zeros.m new file mode 100644 index 0000000..01348be --- /dev/null +++ b/@tfocs_tuple/tfocs_zeros.m @@ -0,0 +1,12 @@ +function x = tfocs_zeros( x ) + +% ABS Absolute value. + +n = numel( x.value_ ); +for k = 1 : n, + x.value_{k} = zeros( size(x.value_{k}) ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/times.m b/@tfocs_tuple/times.m new file mode 100644 index 0000000..56d70b5 --- /dev/null +++ b/@tfocs_tuple/times.m @@ -0,0 +1,9 @@ +function v = times( x, y ) + +% MTIMES Multiplication. + +v = mtimes( x, y ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/@tfocs_tuple/uminus.m b/@tfocs_tuple/uminus.m new file mode 100644 index 0000000..4e31611 --- /dev/null +++ b/@tfocs_tuple/uminus.m @@ -0,0 +1,12 @@ +function x = uminus( x ) + +% UMINUS Unary minus. + +n = numel( x.value_ ); +for k = 1 : n, + x.value_{k} = -x.value_{k}; +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..efa2252 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,229 @@ +Oct 10 2013, v1.3 + License changed! See LICENSE for details + New: + proj_linfl2 Projects rows onto l2 norm constraints + proj_affine.m Projects onto generic affine constraints + proj_singleAffine.m Projects onto single affine constraint + proj_boxAffine.m In response to + http://ask.cvxr.com/question/749/tfocs-projecting-on-box-affine-constraint/: + projects onto intersection of box constraint and affine constraints + proj_conic.m Projects onto the second-order (aka Lorentz) cone + Contributed by Joseph Salmon + proj_l2group.m Projections onto a partitioned l2 group constraint + Contributed by Joseph Salmon + linop_TV3D.m Total variation for 3D grids (linop_TV is for 2D); + contributed by Mahdi Hosseini (mahdi.hosseini@mail.utoronto.ca) + test_LASSO.m Tests the L1RLS (L1 regularized Least-Squares) + problem, aka LASSO + @single/* Allow TFOCS objects to have data type single + May not work well with sparse matrices (which are not allowed + to be sparse data type). Thanks to Graham Coleman. + mexFiles/* New director with mex files + Some non-core functions require mex files, and basic + installation routines and the mex files themselves are in this + directory. + solver_OrderedLASSO.m Solves the ordered LASSO problem + test_OrderedLASSO.m Test script for solver_OrderedLASSO + prox_OL1.m Proximity operator for the ordered LASSO + See http://www-stat.stanford.edu/~candes/OrderedL1/ + "Statistical Estimation and Testing via the Ordered l1 Norm" + by M. Bogdan, E. van den Berg, W. Su, and E. J. Candès, 2013 + mexFiles/proxAdaptiveL1Mex.c Mex file helper for the ordered LASSO + + Modified: + proj_maxEig.m Fixed bug with variable names + prox_l1linf.m Fixed bug with typo of repmat for some cases + prox_nuclear.m Removed dependence on ismatrix() for greater compatibility + @double/tfocs_dot.m Fixed bug for multi-dimensional arrays, and generally made + the code better. Thanks to Graham Coleman for finding the bug. @single/tfocs_dot.m + is similarly updated. + smooth_quad.m Added use_eig mode that makes an expensive + one-time calculation and all subsequent calculations are cheaper + tfocs_initialize.m Fixed bug in initialization code for rounding dimensions; added new + file private/round.m. This affected the demo for alternating completion. + Also allows data to be single data type instead of double (thanks to Graham Coleman) + proj_psdUTrace.m Allows optional constraints that matrix is real-valued. Nov 23 2012 + Can now use eigs for large-scale computation. Feb 15 2013 + Fixed bug in eigs, March 18 2013 + proj_psd.m Modified to allow eigs() version. Feb 15 2013. + prox_trace.m Allows optional constraints that specify the matrix is real-valued. Dec 9 2012 + Fixed bug with eigs parameters in largescale mode + linop_TV.m Implemented norms() function so it is now independent of CVX + Also, adjoint operation is much faster due to a data locality trick + proj_l2.m Fixed bad documentation in .m file. + Also, fixed a bug, so we now project onto the norm ball, not its boundary + Thanks to Graham Coleman, fixed a bug for multidimensional arrays + prox_l1l2.m Using bsxfun for speed improvement + tfocs_SCD.m lines 41-44 were commented out but shouldn't have been; + thanks to Mark Harfouche for noticing this on the ask.cvxr.com forums + + Modified for Octave compatibility: + private/tfocs_prox.m + prox_l1.m + prox_l1l2.m + prox_hingeDual.m + prox_hinge.m + prox_l1linf.m + prox_l1pos.m + prox_nuclear.m + prox_trace.m + proj_maxEig.m + proj_psd.m + + +Sept 7 2012, v1.2 + New: + proj_nuclear.m Projection onto nuclear norm ball. + proj_spectral.m Projection onto spectral norm ball. + prox_max.m Maximum. Dual of proj_simplex.m + proj_max.m Projection so max <= 1. Dual of prox_l1pos.m + prox_maxEig.m Maximum eigenvalue. Dual of proj_psdUTrace.m + proj_maxEig.m Projection so max eig <= 1. Dual of prox_trace.m + prox_dualize.m Computes the Legendre dual of a function. + examples/demos/ Directory with demos (featured on website) is + added + test_TraceLS.m Added. Shows how to run solver_TraceLS.m + test_psdCompletion.m Added. Shows how to run solver_psdComp.m and + solver_psdCompConstrainedTrace.m + Contents.m Added. Describes the relevant functions. + + Modified private/tfocs_prox.m, prox_l1.m, and tfocs_SCD.m to allow + mu to be a vector. Need to modify other prox functions. + Modified test_proxPair.m to test with symmetric, psd, and sparse matrices. + Modified prox_spectral.m to take advantage of symmetric matrices + Modified linop_subsample.m to work in more situations + Modified proj_psd.m to use eigs() if requested, though this is usually not + beneficial + + Fixed bug in prox_hinge.m (for y = [] case ). + Fixed bug in private/tfocs_iterate.m for stopCrit = Inf case + Thanks to Masoud Ahookhosh for finding this. + Fixed bugs in solver_psdComp.m and solver_psdCompConstrainedTrace.m + Made proj_l1.m more numerically stable; thanks to Chris Kauffman for finding + and fixing this. + Made size_compat.m allow multiple singleton dimensions in ND arrays; thanks + to Graham Coleman for finding and fixing this. + +Feb 29 2012, v1.1a + Minor bug fixes, thanks to Graham Coleman for finding them: + + private/print_cell_size.m incorrectly displayed the size of inputs (only + applied in "debug" mode) + private/size_compat.m has been fixed to work with 3D (or ND) arrays + when the final array dimension is a singleton (e.g. 20 x 30 x 1) + +Jan 25 2012, v1.1 + User guide updated. + Changes to code: + New: + linop_fft.m FFT and its transpose. Supports sub-sampling. + test_proxPair.m Tests whether f and fDual are really duals. Not yet + documented in user guide, but see the help text. + solver_sLP_box.m LP with equality and box constraints + test_sBPDN_nonnegative.m Tests BPDN using x >= 0 constraint. + image_denoising_withSPOT.m An example of image denoising + and using the SPOT toolbox. The helper file "plotNow.m" is + also new, and allows you to watch a movie in real-time of + the iterates. Also shows how to use reweighting. + prox_l1pos.m proximity operator for ||x||_1 restricted to x >= 0 + test_SVM.m demo with support vector machines and hinge-loss + test_complicatedUsage.m demo with several matrix variables and + other complicated terms + test_all.m runs all small scale examples + linop_reshape.m reshapes input (an extension of linop_vec). Thanks + to Graham Coleman for contributing. + tfocs.m "debug" option added to main tfocs routine + + Bug fixes: + solver_sSDP.m fixed bug, thanks to Brian Borchers. + prox_hingeDual.m fixed NaN bug + continuation.m fixed bug for case when there are multiple matrix + variables, and for 3D arrays (thanks to Graham Coleman). + prox_spectral.m fixed bugs + test_sTV_largescale.m fixed myAwgn() bug (thanks to Matthew Suttinger) + linop_stack.m fixed bug that occurs when domain is a set of + matrices rather than vectors; thanks to Graham Coleman for discovering. + all solvers: fixed a bug with variable "L" that arose whenever + there is a function called "L.m" in the path (for example, WaveLab has + such a function). + tfocs_LLM.m fixed bug that occurs when restart is used + tfos_initialize.m fixed bug for strong convexity case when Lexact not + specified + prox_boxDual.m fixed bug with bounds were not scalars + proj_l2.m fixed bug for q ~= 1 case + proj_0.m fixed bug for non-constant offsetsl thanks to Graham + Colemen + + Improvements: + tfocs.m allows new "debug" flag that gives more verbose + information, useful when debugging + tfocs_inizialize.m gives more useful error message when sizes are + incorrect + tfocs_iterate.m Improved performance for stopCrit = 3 + tfocs_iterate.m Now supports 'printStopCrit' to display progress of + whatever value is used to determine the stopping criteria + linop_test.m Supports conjugate-symmetric complex inputs/outputs + test_sBPDN_withContinuation.m Updated + solver_sBP.m Supports non-negativity constraints + solver_sBPDN.m Supports non-negativity constraints + tfocs_AT.m Now includes cntr_reset field, which explicitly + recalculates some quantities every so often to avoid accumulation + of roundoff error. + proj_boxDual.m --> prox_boxDual.m to have more consistent naming. + proj_l2.m handles diagonal scaling term; this feature is + experimental and uses a 1D optimization routine. It should + be efficient for N <= 2^18 at least + prox_l2.m same experimental modification as in proj_l2.m + linop_scale.m allows user to specify size explicitly, if desired + prox_hinge.m and prox_hingeDual.m allow more general form with "y" + variable + example_{LMI, LinearProgram, SDP} renamed to test_{LMI, LinearProgram, + SDP } + + +March 20 2011, v1.0c + An almost comprehensive lists of changes since December 2010: + New: + solver_TraceLS.m Solves trace-regularized least-squares problem + solver_sLP.m Linear Program solver + solver_sSDP.m Semi-Definite Program solver + solver_sLMI.m Linear Matrix Inequality solver + prox_hinge.m Proximity fcn for hinge-loss + prox_hingeDual.m Proximity fcn for dual of hinge-loss + proj_0.m Projection onto zero. Added for completeness + proj_boxDual.m Proximity fcn for dual of prox_box.m + smooth_huber.m Huber function + smooth_logLLogistic.m Log-likelihood of the logistic function + smooth_logLPoisson.m Log-likelihood of independent Poisson r.v. + + The continuation feature is now builtin to tfocs_SCD.m + + Updated and/or bug fixes: + linop_subsample.m -- can now handle matrix entry sampling + proj_l2.m -- bug fix + prox_l1.m -- allow scaling "q" to be a non-negative vector + proj_box.m -- bug fix + private/tfocs_initialize.m -- bug fixes + private/tfocs_iterate.m -- bug fixes + + smallscale/examples: the .mat files have been moved to a separate + directory + smooth_quad.m -- allows nonsmooth usage too now + solver_L1RS.m -- bug fix (typo); thanks to Ewout van den Berg + linop_test.m -- now compatible with multidemensional arrays + + New demos: + smallscale/example_LinearProgram.m linear programming + smallscale/example_SDP.m semi-definite programming + smallscale/example_LMI.m linear matrix inequality + largescale/image_denosing_withSPOT.m + + User guide: + Updated to mention the new routines + Describes scaling issues + Describes continuation + Added acknowledgements section + + Misc: + Thanks to Graham Coleman for bug-fixes related to multidimensional + arrays diff --git a/Contents.m b/Contents.m new file mode 100644 index 0000000..8b0a581 --- /dev/null +++ b/Contents.m @@ -0,0 +1,113 @@ +% TFOCS: Templates for First-Order Conic Solvers +% TFOCS v1.3 +%10-Oct-2013 +% +% Main TFOCS program +% tfocs - Minimize a convex problem using a first-order algorithm. +% tfocs_SCD - Smoothed conic dual form of TFOCS, for problems with non-trivial linear operators. +% continuation - Meta-wrapper to run TFOCS_SCD in continuation mode. +% Miscellaneous functions +% tfocs_version - Version information. +% tfocs_where - Returns the location of the TFOCS system. +% Operator calculus +% linop_adjoint - Computes the adjoint operator of a TFOCS linear operator +% linop_compose - Composes two TFOCS linear operators +% linop_scale - Scaling linear operator. +% prox_dualize - Define a proximity function by its dual +% prox_scale - Scaling a proximity/projection function. +% tfunc_scale - Scaling a function. +% tfunc_sum - Sum of functions. +% tfocs_normsq - Squared norm. +% linop_normest - Estimates the operator norm. +% Linear operators +% linop_matrix - Linear operator, assembled from a matrix. +% linop_dot - Linear operator formed from a dot product. +% linop_fft - Fast Fourier transform linear operator. +% linop_TV - 2D Total-Variation (TV) linear operator. +% linop_TV3D - 3D Total-Variation (TV) linear operator. +% linop_handles - Linear operator from user-supplied function handles. +% linop_spot - Linear operator, assembled from a SPOT operator. +% linop_reshape - Linear operator to perform reshaping of matrices. +% linop_subsample - Subsampling linear operator. +% linop_vec - Matrix to vector reshape operator +% Projection operators (proximity operators for indicator functions) +% proj_0 - Projection onto the set {0} +% proj_box - Projection onto box constraints. +% proj_l1 - Projection onto the scaled 1-norm ball. +% proj_l2 - Projection onto the scaled 2-norm ball. +% proj_linf - Projection onto the scaled infinity norm ball. +% proj_linfl2 - Projection of each row of a matrix onto the scaled 2-norm ball. +% proj_max - Projection onto the scaled set of vectors with max entry less than 1 +% proj_conic - Projection onto the second order (aka Lorentz) cone +% proj_l2group - Projection of each group of coordinates onto the 2-norm ball. +% proj_singleAffine - Projection onto a single affine equality or in-equality constraint. +% proj_boxAffine - Projection onto a single affine equality along with box constraints. +% proj_affine - Projection onto a general affine equation, e.g., solutions of linear equations. +% proj_nuclear - Projection onto the set of matrices with nuclear norm less than or equal to q. +% proj_psd - Projection onto the positive semidefinite cone. +% proj_psdUTrace - Projection onto the positive semidefinite cone with fixed trace. +% proj_Rn - "Projection" onto the entire space. +% proj_Rplus - Projection onto the nonnegative orthant. +% proj_simplex - Projection onto the simplex. +% proj_spectral - Projection onto the set of matrices with spectral norm less than or equal to q +% proj_maxEig - Projection onto the set of symmetric matrices with maximum eigenvalue less than 1 +% Proximity operators of general convex functions +% prox_0 - The zero proximity function: +% prox_boxDual - Dual function of box indicator function { l <= x <= u } +% prox_hinge - Hinge-loss function. +% prox_hingeDual - Dual function of the Hinge-loss function. +% prox_l1 - L1 norm. +% prox_Ol1 - Ordered L1 norm. +% prox_l1l2 - L1-L2 block norm: sum of L2 norms of rows. +% prox_l1linf - L1-LInf block norm: sum of L2 norms of rows. +% prox_l1pos - L1 norm, restricted to x >= 0 +% prox_l2 - L2 norm. +% prox_linf - L-infinity norm. +% prox_max - Maximum function. +% prox_nuclear - Nuclear norm. +% prox_spectral - Spectral norm, i.e. max singular value. +% prox_maxEig - Maximum eigenvalue of a symmetri matrix. +% prox_trace - Nuclear norm, for positive semidefinite matrices. Equivalent to trace. +% Smooth functions +% smooth_constant - Constant function generation. +% smooth_entropy - The entropy function -sum( x_i log(x_i) ) +% smooth_handles - Smooth function from separate f/g handles. +% smooth_huber - Huber function generation. +% smooth_linear - Linear function generation. +% smooth_logdet - The -log( det( X ) ) function. +% smooth_logLLogistic - Log-likelihood function of a logistic: sum_i( y_i mu_i - log( 1+exp(mu_i) ) ) +% smooth_logLPoisson - Log-likelihood of a Poisson: sum_i (-lambda_i + x_i * log( lambda_i) ) +% smooth_logsumexp - The function log(sum(exp(x))) +% smooth_quad - Quadratic function generation. +% Testing functions +% test_nonsmooth - Runs diagnostic tests to ensure a non-smooth function conforms to TFOCS conventions +% test_proxPair - Runs diagnostics on a pair of functions to check if they are Legendre conjugates. +% test_smooth - Runs diagnostic checks on a TFOCS smooth function object. +% linop_test - Performs an adjoint test on a linear operator. +% Premade solvers for specific problems (vector variables) +% solver_L1RLS - l1-regularized least squares problem, sometimes called the LASSO. +% solver_LASSO - Minimize residual subject to l1-norm constraints. +% solver_OrderedLASSO - LASSO using ordered l1-norm. +% solver_sBP - Basis pursuit (l1-norm with equality constraints). Uses smoothing. +% solver_sBPDN - Basis pursuit de-noising. BP with relaxed constraints. Uses smoothing. +% solver_sBPDN_W - Weighted BPDN problem. Uses smoothing. +% solver_sBPDN_WW - BPDN with two separate (weighted) l1-norm terms. Uses smoothing. +% solver_sDantzig - Dantzig selector problem. Uses smoothing. +% solver_sDantzig_W - Weighted Dantzig selector problem. Uses smoothing. +% solver_sLP - Generic linear programming in standard form. Uses smoothing. +% solver_sLP_box - Generic linear programming with box constraints. Uses smoothing. +% Premade solvers for specific problems (matrix variables) +% solver_psdComp - Matrix completion for PSD matrices. +% solver_psdCompConstrainedTrace - Matrix completion with constrained trace, for PSD matrices. +% solver_TraceLS - Unconstrained form of trace-regularized least-squares problem. +% solver_sNuclearBP - Nuclear norm basis pursuit problem (i.e. matrix completion). Uses smoothing. +% solver_sNuclearBPDN - Nuclear norm basis pursuit problem with relaxed constraints. Uses smoothing. +% solver_sSDP - Generic semi-definite programs (SDP). Uses smoothing. +% solver_sLMI - Generic linear matrix inequality problems (LMI is the dual of a SDP). Uses smoothing. +% Algorithm variants +% tfocs_AT - Auslender and Teboulle's accelerated method. +% tfocs_GRA - Gradient descent. +% tfocs_LLM - Lan, Lu and Monteiro's accelerated method. +% tfocs_N07 - Nesterov's 2007 accelerated method. +% tfocs_N83 - Nesterov's 1983 accelerated method; also by Beck and Teboulle 2005 (FISTA). +% tfocs_TS - Tseng's modification of Nesterov's 2007 method. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c708006 --- /dev/null +++ b/LICENSE @@ -0,0 +1,34 @@ +Copyright (c) 2013, California Institute of Technology and CVX Research, Inc. +All rights reserved. + +Contributors: Stephen Becker, Emmanuel Candes, and Michael Grant. +Based partially upon research performed under the DARPA/MTO Analog-to- +Information Receiver Development Program, AFRL contract #FA8650-08-C-7853. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the names of California Institute of Technology + (Caltech), CVX Research, Inc., nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/README b/README new file mode 100644 index 0000000..8183d0f --- /dev/null +++ b/README @@ -0,0 +1,11 @@ +Templates for First-Order Conic Solvers (TFOCS) +Version 1.3, October 10, 2013 +by Stephen Becker, Emmanuel Candes, and Michael Grant. +Copyright 2011-2013, California Institute of Technology and CVX Research. + +Please see the file tfocs_userguide.pdf for instructions on how to use TFOCS +and how to submit a support request. + +As of October 2, 2013, TFOCS is now subject to a 3-Clause BSD license. Please +see the file LICENSE for the full text of the license. + diff --git a/continuation.m b/continuation.m new file mode 100644 index 0000000..8e32292 --- /dev/null +++ b/continuation.m @@ -0,0 +1,261 @@ +function [ x, odata, optsOut ] = continuation( fcn, mu, x0, z0, opts, contOpts ) +% CONTINUATION Meta-wrapper to run TFOCS_SCD in continuation mode. +% [...] = CONTINUATION( FCN, MU, X0, Z0, OPTS, CONT_OPTS ) +% is a wrapper to perform continuation on FCN, where +% FCN is a function that calls tfocs_SCD or a tfocs solver. +% FCN must accept input arguments MU, X0, Z0, and OPTS, +% e.g. FCN = @(mu,x0,z0,opts) solver_sBP( A, b, mu, x0, z0, opts ) +% +% CONT_OPTS are options that affect how continuation is performed. +% To see the default options, call this script with no inputs. +% +% Options for CONT_OPTS: +% maxIts - max # of continuation iterations +% accel - use accelerated continuation or not +% betaTol - every continuation iteration, the tolerance +% is decreased by 'betaTol' +% innerTol - every continuation iteration, except the last one, +% is solved to this tolerance. +% This option overrides 'betaTol' +% tol - the outer loop will stop when either +% 'maxIts' is reached, or when the change +% from one step to the next is less than 'tol'. +% If innerTol is not specified, then "tol" and "betaTol" +% are used to determine the tolerance of intermediate solves. +% initialTol - For use with "tol". This overrides betaTol, but in turn is +% overridden by innerTol. If specified, then the first +% problem is solved to tolerance "initialTol" and the final +% problem is solved to tolerance "tol", and all problems +% in between are solved to logarithmically interpolated values. +% E.g. if maxits = 3 and initialTol = .1 and tol = .001, +% then the tolerances are .1, .01, .001. +% finalTol - For use with "tol". If "finalTol" is set, then "tol" +% and "betaTol" are used to determine the tolerance of +% intermediate solves, but the last solve is solved +% to the "finalTol" tolerance. +% innerMaxIts - maximum number of inner iterations during +% every continuation iteration except the last one. +% muDecrement - after every iteration, mu <-- mu*muDecrement +% (default: muDecrement = 1, so mu is constant) +% verbose - true (default) prints out information, false will +% not print out any information +% +% As of release 1.1, you do not need to call this file directly. Instead, +% you can specify the continuation option in TFOCS_SCD.m directly. +% See also TFOCS_SCD.m + +if nargin < 6, contOpts = []; end +kMax = setOpts('maxIts',3); +ACCEL = setOpts('accel',true); +betaTol = setOpts('betaTol',2); +innerTol= setOpts('innerTol',[] ); +stopTol = setOpts('tol',1e-3); +finalTol= setOpts('finalTol',[]); +verbose = setOpts('verbose',true); +innerMaxIts = setOpts('innerMaxIts',[] ); +initialTol = setOpts('initialTol',[]); +muDecrement = setOpts('muDecrement',1); +finalRestart = setOpts('finalRestart',[]); % not yet documented. usefulness? + +if nargin == 0 + if nargout == 0 + disp('Default options for CONTINUATION are:'); + end + x = contOpts; + return; +end + +error(nargchk(2,6,nargin)); +if nargin < 3, x0 = []; end +if nargin < 4, z0 = []; end +if nargin < 5, opts = []; end + +% what is the user specified stopping tolerance? +if ~isfield(opts,'tol') + % get the default value: + defaultOpts = tfocs_SCD(); + opts.tol = defaultOpts.tol; +end +if isfield(opts,'fid'), fid = opts.fid; else fid = 1; end + + +finalTol2 = opts.tol; +if isempty( innerTol ) && ~isempty(initialTol) + % calculate the value of betaTol that will + betaTol = exp( log(initialTol/finalTol2)/(kMax-1) ); +end +% tol = finalTol2*betaTol^(kMax+1); % bug fixed Feb 2011 +tol = finalTol2*betaTol^(kMax); + +xOld = x0; +odataCumulative = []; +odataCumulative.niter= 0; +extraVerbose = verbose && (isfield(opts,'printEvery') && (opts.printEvery <= 0 ... + || opts.printEvery == Inf ) ); +for k = 1:kMax + if kMax > 1 && verbose +% fprintf(fid,'--- Continuation step %d of %d ', k, kMax ); + fprintf(fid,'--- Continuation step %d of %d, mu: %.1e', k, kMax, mu ); + if extraVerbose + fprintf(fid,' ...'); + else + fprintf('\n'); + end + end + + optsTemp = opts; + if ~isempty( innerMaxIts ) && k < kMax + optsTemp.maxIts = innerMaxIts; + end + tol = tol/betaTol; + if ~isempty( innerTol ) && k < kMax + optsTemp.tol = innerTol; + else + if ~isempty(finalTol) && k == kMax + optsTemp.tol = finalTol; + else + optsTemp.tol = tol; + end + end + if k == kMax && ~isempty( finalRestart ) + optsTemp.restart = finalRestart; + end + + % call the solver + [x, odata, optsOut ] = fcn( mu, x0, z0, optsTemp ); + + if iscell(x) + % Thanks to Graham Coleman for catching this case + if isempty(x0) + mu2normXX = mu/2*sum(cellfun( @(x) norm(x(:))^2, x )); + elseif iscell(x0) + mu2normXX = mu/2*sum(cellfun( @(x) norm(x(:))^2, ... + cellfun( @minus, x, x0,'uniformoutput',false ) )); + else + mu2normXX = mu/2*sum(cellfun( @(x) norm(x(:))^2, ... + cellfun( @(x) x-x0, x, 'uniformoutput',false ) )); + end + else + if isempty(x0) + mu2normXX = mu/2*norm(x(:))^2; + else + mu2normXX = mu/2*norm(x(:)-x0(:))^2; + end + end + + % (possibly) decrease mu for next solve + mu = mu*muDecrement; + + % update output data + fields = { 'f', 'normGrad', 'stepsize','theta','counts','err' }; + for ff = fields + f = ff{1}; + if isfield(odata,f) + if isfield(odataCumulative,f) + odata.(f) = [odataCumulative.(f); odata.(f)]; + end + odataCumulative.(f) = odata.(f); + end + + end + if isfield(odata,'niter') + % include the old iterations in the count + odataCumulative.niter = odata.niter + odataCumulative.niter; + odata.niter = odataCumulative.niter; + + % record at which iterations continuation happened + if isfield(odataCumulative,'contLocations') + odataCumulative.contLocations = [odataCumulative.contLocations; odata.niter]; + odata.contLocations = odataCumulative.contLocations; + else + odata.contLocations = odata.niter; + odataCumulative.contLocations = odata.niter; + end + end + + if kMax > 1 && extraVerbose + fprintf(fid,'\b\b\b\b'); % remove the " ..." from earlier + end + % if the tfocs solver is set to be quiet, but the continuation + % script still has the "verbose" flag, then print out + % some information + if extraVerbose && isfield( odata, 'err' ) && ~isempty( odata.err ) + fprintf(fid,', errors: '); + fprintf(fid,' %8.2e', odata.err(end,:) ); + end + if verbose + if extraVerbose + fprintf(fid,', mu/2||x-x_0||^2: %.1e', mu2normXX ); + else + fprintf(fid,'Continuation statistics: mu/2||x-x_0||^2: %.1e', mu2normXX ); + end + + if kMax > 1 && extraVerbose + fprintf(fid,' ---'); + end + fprintf(fid,'\n'); + end + + + if k == kMax, break; end + + % Update the prox center + + if ACCEL + if iscell(x) + if isempty(xOld) || ~iscell(xOld) + xOld = cell( size(x) ); + end + x0 = cell( size(x) ); + for nCell = 1:length(x) + if isempty(xOld{nCell}), xOld{nCell} = zeros( size(x{nCell}) ); end + x0{nCell} = x{nCell} + (k-1)/(k+2)*( x{nCell} - xOld{nCell} ); + end + + else + if isempty(xOld), xOld = zeros(size(x)); end + x0 = x + (k-1)/(k+2)*( x - xOld ); + end + else + x0 = x; + end + + if isa( odata.dual, 'tfocs_tuple') + z0 = cell( odata.dual ); + else + z0 = odata.dual; + end + + + if iscell(x) + n1=sqrt(sum(cellfun( @(x) norm(x(:))^2, cellfun( @minus, x, xOld,'uniformoutput',false ) ))); + n2=sqrt(sum(cellfun( @(x) norm(x(:))^2, xOld ))); + else + n1 = norm(x(:)-xOld(:)); + n2 = norm(xOld(:)); + end + + if n1/n2 <= stopTol && verbose + fprintf(fid,'Continuation ending, due to convergence\n'); + break; + end + xOld = x; + if k == kMax && verbose + fprintf(fid,'Continuation ending, due to reaching maximum number of outer iterations\n'); + end +end + + +function out = setOpts(fieldName,default) + if ~isfield(contOpts,fieldName) + contOpts.(fieldName) = default; + end + out = contOpts.(fieldName); +end + + +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/demos/SIAM_demo.m b/examples/demos/SIAM_demo.m new file mode 100644 index 0000000..aa2efe7 --- /dev/null +++ b/examples/demos/SIAM_demo.m @@ -0,0 +1,195 @@ +%% RPCA demo using TFOCS +% This is a demo of Robust PCA (RPCA) using . Since 2009, +% there has been much interest in this specific RPCA formulation +% (RPCA can refer to many different formulations; we will state +% our formulation precisely below), but most solvers +% assume equality constraints. One great feature of TFOCS +% is how easy it is to prototype new formulations, so we show +% here a formulation using inequality constraints. +% +% The basic (equality constraint) RPCA formulation is: +% ( see +% for references ) +% +% $$ \min_{S,L} \|L\|_* + \lambda \|S\|_{\ell_1} \quad\textrm{subject to}\;L+S = X $$ +% +% The idea is that some object $X$ can be broken into two components $X=L+S$, +% where $L$ is low-rank and $S$ is sparse. To tease out this separation, +% RPCA penalizes $\|L\|_*$ (the nuclear norm of $L$), which is a proxy for +% the rank of $L$. We also penalizes $\|S\|_{\ell_1}$ which is just the sum +% of the absolute value of all the entries of the matrix $S$; this is a proxy +% for the number of non-zero entries of $S$. +% +% The parameter $\lambda$ is important, as it controls how much weight +% to put on $S$ relative to $L$. This parameter is not overly sensitive, +% and we picked the value used in this demo by a few quick rounds of trial-and-error. +% +% _Code by Stephen Becker, May 2011_ + +%% Inequality constraints +% In this demo, $X$ is a video. In particular, each column of $X$ is the set +% of $m n$ pixels from a $m \times n$ pixel frame of the video, reshaped +% so that it is now a vector instead of a matrix. +% +% The particular video we use is not compressed using a video codec, so +% each frame of the video is stored. This leads to huge files, so to save +% a bit of space, the value of each pixel is stored as an 8 bit number, +% which means we can think of it as a value from $0,1,2,\ldots,255$. Now, +% we may imagine that there is some "true" video where each pixel +% is a _real_ number between $[0,255]$, and that we see a quantized version of it. +% It would be nice to apply RPCA to this true version instead of applying +% it to the quantized version. In particular, it is unlikely that the quantized +% version can be split nicely into a low-rank and sparse component. +% To account for this quantization, we will ask that $S+L$ is not exactly equal +% to $X$, but rather that $S+L$ agrees with $X$ up to the precision +% of the quantization. The quantization can induce at most an error of .5 in +% the pixel value (e.g. true value is 5.6, which is rounded to 6.0, so an error of .4; +% or a true value of 6.4, which is also rounded to 6.0, so also an error of 0.4). +% A nice way to capture this is via the $\ell_\infty$ norm. +% +% So, the inequality constrained version of RPCA uses the same objective +% function, but instead of the constraints $L+S=X$, the constraints are +% +% $$ \|L+S-X\|_{\ell_\infty} \le 0.5. $$ + +%% Numerical demo +% We demonstrate this on a video clip taken from a surveillance +% camera in a subway station. Not only is there a background +% and a foreground (i.e. people), but there is an escalator +% with periodic motion. Conventional background subtraction +% algorithms would have difficulty with the escalator, but +% this RPCA formulation easily captures it as part of the low-rank +% structure. +% The clip is taken from the data at this website (after +% a bit of processing to convert it to grayscale): +% http://perception.i2r.a-star.edu.sg/bk_model/bk_index.html + +% Load the data: +disp('Please download the escalator_data.m (3.7 MB) file from:') +disp(' http://tfocs.stanford.edu/demos/rpca/escalator_data.mat'); +load escalator_data % contains X (data), m and n (height and width) +X = double(X); + +% addpath ~/Dropbox/TFOCS/ % add TFOCS to your path, so change this to suit your computer + + +%% +nFrames = size(X,2); + +lambda = 1e-2; + +opts = []; +opts.stopCrit = 4; +opts.printEvery = 1; +opts.tol = 1e-4; + +opts.maxIts = 25; + +opts.errFcn{1} = @(f,d,p) norm(p{1}+p{2}-X,'fro')/norm(X,'fro'); + +largescale = false; + +for inequality_constraints = 0:1 + + + if inequality_constraints + % if we already have equality constraint solution, + % it would make sense to "warm-start": +% x0 = { LL_0, SS_0 }; + % but it's more fair to start all over: + x0 = { X, zeros(size(X)) }; + z0 = []; + else + x0 = { X, zeros(size(X)) }; + z0 = []; + end + + + obj = { prox_nuclear(1,largescale), prox_l1(lambda) }; + affine = { 1, 1, -X }; + + mu = 1e-4; + if inequality_constraints + epsilon = 0.5; + dualProx = prox_l1(epsilon); + else + dualProx = proj_Rn; + end + + tic + % call the TFOCS solver: + [x,out,optsOut] = tfocs_SCD( obj, affine, dualProx, mu, x0, z0, opts); + toc + + % save the variables + LL =x{1}; + SS =x{2}; + if ~inequality_constraints + z0 = out.dual; + LL_0 = LL; + SS_0 = SS; + end + +end % end loop over "inequality_constriants" variable + +%% show all together in movie format +% If you run this in your own computer, you can see the movie. On the webpage, +% we have a youtube version of the video. +% The top row is using equality constraints, and the bottom row +% is using inequality constraints. +% The first column of both rows is the same (i.e. it is the original image). +mat = @(x) reshape( x, m, n ); +figure(); +colormap( 'Gray' ); +k = 1; +for k = 1:nFrames + + imagesc( [mat(X(:,k)), mat(LL_0(:,k)), mat(SS_0(:,k)); ... + mat(X(:,k)), mat(LL(:,k)), mat(SS(:,k)) ] ); + + axis off + axis image + + drawnow; + pause(.05); + + if k == round(nFrames/2) + snapnow; % Take a single still snapshot for publishing the m file to html format + end +end + +%% Is there a difference between the two versions? +% Compare the equality constrained version and the inequality +% constrained version. To do this, we can check whether +% the "L" components are really low rank +% and whether the "S" components are really sparse. +% Even if the two versions appear visually similar, the variables +% may behave quite differently with respect to low-rankness +% and sparsity. + +fprintf('"S" from equality constrained version has \t%.1f%% nonzero entries\n',... + 100*nnz(SS_0)/numel(SS_0) ); +fprintf('"S" from inequality constrained version has \t%.1f%% nonzero entries\n',... + 100*nnz(SS)/numel(SS) ); + +s = svd(LL); % inequality constraints +s0 = svd(LL_0); % equality constraints + +fprintf('"L" from equality constrained version has numerical rank\t %d (of %d possible)\n',... + sum( s0>1e-6), min( m*n, nFrames) ); +fprintf('"L" from inequality constrained version has numerical rank\t %d (of %d possible)\n',... + sum( s > 1e-6), min( m*n, nFrames) ); + +figure(); +semilogy( s0 ,'o-') +hold all; +semilogy( s ,'o-') +legend('Equality constraints','Inequality constraints'); +xlabel('sorted singular value location'); +ylabel('value of singular value'); +title('Comparison of RPCA using equality and inequality constraints'); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/examples/demos/callandmap.m b/examples/demos/callandmap.m new file mode 100644 index 0000000..92b5f3a --- /dev/null +++ b/examples/demos/callandmap.m @@ -0,0 +1,15 @@ +function varargout = callandmap(fcn, ix, varargin) +%CALLANDMAP Call a function and rearrange its output arguments +% varargout = callandmap( fcn, ix, varargin ) +% +% Suggested here: +% http://stackoverflow.com/questions/3673392/define-anonymous-function-as-2-of-4-outputs-of-m-file-function + +tmp = cell(1,max(ix)); % Capture up to the last argout used +[tmp{:}] = fcn(varargin{:}); % Call the original function +varargout = tmp(ix); % Remap the outputs + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/examples/demos/demo_MatrixCompletion.m b/examples/demos/demo_MatrixCompletion.m new file mode 100644 index 0000000..6539035 --- /dev/null +++ b/examples/demos/demo_MatrixCompletion.m @@ -0,0 +1,69 @@ +%% Matrix completion demo +% This short demo shows how to use TFOCS to perform nuclear norm minimization. +% Nuclear norm minimization is used for recovering all the entries of +% a partially observed low-rank matrix. + +%% Setup a problem +rng(234923); % for reproducible results +N = 16; % the matrix is N x N +r = 2; % the rank of the matrix +df = 2*N*r - r^2; % degrees of freedom of a N x N rank r matrix +nSamples = 3*df; % number of observed entries + +% For this demo, we will use a matrix with integer entries +% because it will make displaying the matrix easier. +iMax = 5; +X = randi(iMax,N,r)*randi(iMax,r,N); % Our target matrix + +%% +% Now suppose we only see a few entries of X. Let "Omega" be the set +% of observed entries +rPerm = randperm(N^2); % use "randsample" if you have the stats toolbox +omega = sort( rPerm(1:nSamples) ); + +%% +% Print out the observed matrix in a nice format. +% The "NaN" entries represent unobserved values. The goal +% of this demo is to find out what those values are! + +Y = nan(N); +Y(omega) = X(omega); +disp('The "NaN" entries represent unobserved values'); +disp(Y) + +%% Matrix completion via TFOCS +% We use nuclear norm relaxation. There are strong theorems that +% show this relaxation will usually give you the *exact* original low-rank +% matrix provided that certain conditions hold. However, these +% conditions are generally not possible to check 'a priori', +% so I cannot guarantee that this will work. But by choosing +% enough measurements, it becomes increasingly likely to work. + +% Add TFOCS to your path (modify this line appropriately): +addpath ~/Dropbox/TFOCS/ + +observations = X(omega); % the observed entries +mu = .001; % smoothing parameter + +% The solver runs in seconds +tic +Xk = solver_sNuclearBP( {N,N,omega}, observations, mu ); +toc + +%% +% To display the recovered matrix, let's round it to the nearest +% .0001 so that it displays nicely: +disp('Recovered matrix (rounding to nearest .0001):') +disp( round(Xk*10000)/10000 ) + +% and for reference, here is the original matrix: +disp('Original matrix:') +disp( X ) + +% The relative error (without the rounding) is quite low: +fprintf('Relative error, no rounding: %.8f%%\n', norm(X-Xk,'fro')/norm(X,'fro')*100 ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/examples/demos/demo_SVM.m b/examples/demos/demo_SVM.m new file mode 100644 index 0000000..65bfda9 --- /dev/null +++ b/examples/demos/demo_SVM.m @@ -0,0 +1,147 @@ +%% Support Vector Machine (SVM) example +% +% We have binary data, and the two classes are labeled +1 and -1. +% The data is d-dimensional, and we have n samples +% +% +% This example show show to solve the standard SVM using the hinge-loss +% and l2 penalty. Then it shows how it is easy with TFOCS to generalize +% the SVM problem and use the l1 penalty instead of the l2 penalty, +% which has the effect of encouraging sparsity. +% +% For a more detailed example, see the /examples/largescale/demo_SVM.m +% code in the TFOCS release. + +% Before running this, please add the TFOCS base directory to your path +addpath ~/Dropbox/TFOCS/ + +%% Generate a new problem + +randn('state',23432); +rand('state',3454); + +n = 30; +d = 2; % 2D data is nice because it's easy to visualize + +n1 = round(.5*n); % number of -1 (this is data from one "class") +n2 = n - n1; % number of +1 (this is data from the other "class") + +% Generate data +mean1 = [-.5,1]; +% mean2 = [.6,.2]; % can be separated +mean2 = [.1,.5]; % cannot be separated, so a bit more interesting +s = .25; % standard deviation +x1 = repmat(mean1,n1,1) + s*randn(n1,2); +x2 = repmat(mean2,n2,1) + s*randn(n2,2); + +X = [x1; x2]; +labels = [ ones(n1,1); -ones(n2,1) ]; + +% Plot the data: +figure(1); clf; +hh = {}; +hh{1}=plot(x1(:,1), x1(:,2), 'o' ); +hold all +hh{2}=plot(x2(:,1), x2(:,2), '*' ); +legend('Class 1','Class 2'); +xl = get(gca,'xlim'); +yl = get(gca,'ylim'); +legend([hh{1:2}],'Class 1','Class2'); +%% Introduce the SVM problem formulations +% The hinge-loss can be defined in Matlab as: +hinge = @(x) sum(max(0,1-x)); +%% +% To see what it looks like: +grid = linspace(-2,2,30); +figure(2); +plot( grid, max(0,1-grid) ) +ylim( [-.5,3] ); set(gcf,'Position',[100,100,350,250] ); +%% +% Then the standard SVM in primal form is the following: +% +% minimize_{m,b} hinge( labels.*( X*m - b ) ) + lambda_A*norm(m,2) +% +% where X is the data matrix (dimensions n x d ), m is the slope (dimension d x 1 ), +% and b is an offset (dimension 1 for our example). +% +% To put this in a more amenable format, introduce the variable "a" +% and let a = [m;b], so "a" has dimensions (d+1) x 1. +% Then we can express "X*m-b" as "[X,-ones(n,1)]*a". +% For this reason, we introduce the following linear operator: +linearF = diag(labels)*[ X, -ones(n,1) ]; + +%% +% So now, the problem is: +% +% minimize_{a} hinge( linearF*a ) + lambda_A*norm( [ones(d,1); 0] * a ) +% +% (in the "norm" term, we want a 0 term in front of a(d+1) since we +% do not wish to penalize the norm of the constant offset). +% +% A reasonable value of lambda_A is: +lambda_A = 10; + +%% Introduce the sparse SVM formulation +% The sparse SVM is designed to induce sparsity in the slope variable +% m by replacing the lambda_A*norm(m,2) term with a lambda_B*norm(m,1) +% term, since the l1 term drives small coefficients to zero. +% Here's a reasomable value for lambda_B: +lambda_B = 4; + +%% Solve SVM +mu = 1e-2; % smoothing parameter +%% +% Dualize the hinge loss and the l2 terms: +prox = { prox_hingeDual(1,1,-1), proj_l2(lambda_A) }; +%% +% Make the affine operator: +offset1 = []; +linearF2 = diag( [ones(d,1);0] ); +offset2 = []; +affineF = { linearF, offset1; linearF2, offset2 }; +ak = tfocs_SCD([],affineF, prox, mu); + +%% +% Plot +m = ak(1:2); % slope +b = ak(3); % intercept +grid = linspace(-.5,1,100); +figure(1); +hh{3} = plot( grid, (b-m(1)*grid)/m(2) ); +xlim(xl); ylim(yl); +legend([hh{1:3}],'Class 1','Class2','SVM'); + +%% Solve Sparse SVM +PROBLEM = 2; +mu = 1e-2; % smoothing parameter +opts = []; +opts.tol = 1e-3; +%% +% Dualize the hinge loss to get 'prox_hingeDual' +% For the hingeDual, scale by -1 since we want polar cone, not dual cone +% Dualize the l1 term to get the linf term +prox = { prox_hingeDual(1,1,-1), proj_linf(lambda_B) }; +linearF2 = diag( [ones(d,1);0] ); +ak = tfocs_SCD([],{linearF,[];linearF2,[]}, prox, mu,[],[],opts); + +%% +% Plot +m = ak(1:2); % slope +b = ak(3); % intercept +grid = linspace(-.5,1,100); +hh{4} = plot( grid, (b-m(1)*grid)/m(2) ); +xlim(xl); ylim(yl); +legend([hh{:}],'Class 1','Class2','SVM','sparse SVM'); + +%% +% You can see that the sparse SVM hyperplane is vertical! This is because +% only the x-component is zero, and only the y-component is non-zero. +% So it is "sparse" (though in this 2D example, "sparsity" doesn't mean +% much). For larger dimensional data, the idea is that identifying +% a hyperplane that only has a few non-zeros will allow you to identify +% which dimensions are actually important. + +%% +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/demos/demo_alternatingProjections.m b/examples/demos/demo_alternatingProjections.m new file mode 100644 index 0000000..5af800d --- /dev/null +++ b/examples/demos/demo_alternatingProjections.m @@ -0,0 +1,402 @@ +%% Projection onto the intersection of two sets +% Our goal is to find a point in the intersection of two convex sets C1 and C2. +% A similar goal is to find the projection of an arbitrary point onto the +% intersection (or if the intersection is empty, find the point(s) in C1 +% that are as close as possible to C2). In this demo, the intersection +% of C1 and C2 is a singleton, so the concepts are the same. +% +% We assume that we know how to project on the sets C1 and C2 individually, +% but not onto C1 intersect C2. +% +% The basic algorithm to solve this problem is the alternating projection method, +% first studied by John von Neumann for the case where C1 and C2 are affine +% spaces. This algorithm can be extended to arbitrary convex sets, although +% you may not converge to the *projection* of the original point. +% +% This algorithm is very simple: starting at some point y, and with x <-- y, +% we update +% x <-- Proj_{C1}( Proj_{C2}( x ) ) +% where Proj_{C1} is the projector onto the set C1. +% +% Better methods (which actually give you the projection of y onto the intersection) +% are based on Dykstra's algorithm (not to be confused with the Dijkstra algorithm +% described here: http://en.wikipedia.org/wiki/Dijkstra%27s_algorithm ). +% +% Dykstra's algorithm also uses only Proj_{C1} and Proj_{C2}, but with a few +% intermediate steps. For an overview, see the 2011 book by Raydan: +% http://www.amazon.com/Alternating-Projection-Methods-Fundamentals-Algorithms/dp/1611971934 +% +% Another overview of both alternating projection and Dykstra are in the book chapter +% "Proximal splitting methods in signal processing" by P. L. Combettes and J.-C. Pesquet, +% in the book 'Fixed-Point Algorithms for Inverse Problems in Science and Engineering', +% (ed.: H. H. Bauschke, R. S. Burachik, P. L. Combettes, V. Elser, D. R. Luke, and +% H. Wolkowicz, Editors), pp. 185-212. Springer, New York, 2011. +% The chapter can be downloaded at http://www.ann.jussieu.fr/~plc/prox.pdf +% +% The purpose of this demo is to show the above two methods, and also +% how to formulate this with TFOCS. The advantage of solving this with TFOCS +% is that we can use an accelerated solver (Nesterov-style) and use large +% step-sizes and line search (whereas Dykstra has no step-size parameter). +% In the first example, it is especially apparent that TFOCS avoids +% the slow convergence that both the alternating projection and Dykstra +% algorithms suffer from. In the second example, the performance +% of alternating projections and TFOCS are quite similar. + +%% Mathematical formulation +% For a given point y, and two convex sets C1 and C2, we wish to find +% the closest point x to y, such that x is in both C1 and C2. We write this +% as: +% +% $$ \textrm{minimize}_{x\in C_1 \cap C_2} \,\, \mu/2||x\textrm{--}y||_2^2 $$ +% +% The parameter $\mu > 0$ is arbitrary and does not affect the answer. +% +% This fits naturally into the TFOCS formulation since the primal +% problem is already strongly convex. + + + +%% Demo with two polygon sets in 2D that intersect at (0,0) +% Set the dimension to 2 +N = 2; % 2D is best for visualization + +%% +% Define some 2D polygons + +mx = 1; +x1a = [1;.1]; +x1b = [1;.2]; +xx1 = [0,x1a(1),2*mx,x1b(1)]; +yy1 = [0,x1a(2),2*mx,x1b(2)]; + + +x2a = [1;.3]; +x2b = [1;.35]; +xx2 = [0,x2a(1),2*mx,x2b(1)]; +yy2 = [0,x2a(2),2*mx,x2b(2)]; + +%% +% The solution is at x = (0,0), so our error function is simple + +err = @(x) norm(x); +%% +% plot the regions + +figure +red = [255,153,153]/255; +blue = [204,204,255]/255; +fill( xx1, yy1,red); +hold on +fill( xx2, yy2,blue); + +xlim( [0,mx] ); ylim( [0,.5*mx] ); +axis equal +text(.5,.07,'set C1'); +text(.5,.22,'set C2'); + +%% +% Make some operators that will be used with TFOCS + +addpath ~/Dropbox/TFOCS/ % modify this to wherever it is installed on your computer +op1 = project2DCone(x1a, x1b ); +op2 = project2DCone(x2a, x2b ); + +offset1 = 0; +offset2 = 0; +op1_d = prox_dualize(op1); +op2_d = prox_dualize(op2); + +% and some simpler operators that we will use with alternating projections +% and with Dykstra +proj1 = @(x) callandmap( op1, 2, x, 1); % gamma is irrelevant +proj2 = @(x) callandmap( op2, 2, x, 1 ); % gamma is irrelevant + +% for all methods, we need to specify a starting point + +x0 = [1; 1/4]; + + +%% solve in TFOCS +% We can pick any mu and should get the same result, although due to +% some scaling issues in stopping criteria, it does have a small effect. +mu = 1e-6; + +opts = struct('debug',false,'printEvery',20,'maxIts',200); +opts.errFcn{1} = @(f,d,x) err(x); +opts.errFcn{2} = @(f,d,x) recordPoints(x); +recordPoints(); % zero-out counter. We use this to plot the points later + +opts.tol = 1e-15; + +affineOperator = {eye(N),offset1;eye(N),offset2}; +dualProx = {op1_d,op2_d}; +% Solve in TFOCS: +[x,out,optsOut] = tfocs_SCD( [], affineOperator, dualProx , mu, x0, [], opts ); + +% Record the path: +path = recordPoints(); +path = [x0,path]; + +figure +subplot(1,2,1); +fill( xx1, yy1,red); +hold on +fill( xx2, yy2,blue); +xlim( [0,mx] ); ylim( [0,.5*mx] ); +text(.5,.07,'set C1'); +text(.5,.22,'set C2'); +plot( path(1,:), path(2,:),'ko-','linewidth',2 ) +title('Path for TFOCS solving the intersection of 2 polygons'); + +subplot(1,2,2); +semilogy( out.err(:,1) ,'o-'); xlabel('iterations'); ylabel('error'); +title('Error for TFOCS method'); +set(gcf, 'Position', [400 200 800 400]); +%% solve via alternating projection method +maxIts = 500; +x = x0; +path = [x0]; +errHist = []; +for k = 1:maxIts + % basic alternating projection method: + x = proj1(x); + path= [path, x]; + x = proj2(x); + path= [path, x]; + errHist = [ errHist; err(x) ]; + if ~mod(k,50) + fprintf('Iter %4d, error is %.2e\n', k, errHist(end) ); + end +end +figure +subplot(1,2,1); +fill( xx1, yy1,red); +hold on +fill( xx2, yy2,blue); +xlim( [0,mx] ); ylim( [0,.5*mx] ); +text(.5,.07,'set C1'); +text(.5,.22,'set C2'); +plot( path(1,:), path(2,:),'ko-','linewidth',1 ) +title('Path for alternating projection method solving the intersection of 2 polygons'); + +subplot(1,2,2); +semilogy( errHist, 'o-' ); xlabel('iterations'); ylabel('error'); +errHist_1 = errHist; +title('Error for alternating projection method'); +set(gcf, 'Position', [400 200 800 400]); +%% solve via Dykstra's algorithm + +maxIts = 500; +x = x0; +[p,q] = deal( 0*x0 ); +p = -.25*[1;1]; +path = [x0]; +errHist = []; +for k = 1:maxIts + % If x + p is feasible, the y = (x+p), so p=x+p-y = 0. + y = proj1( x + p ); + p = x + p - y; + x = proj2( y + q ); + q = y + q - x; + path = [path,y,x]; + errHist = [ errHist; err(x) ]; + if ~mod(k,50) + fprintf('Iter %4d, error is %.2e\n', k, errHist(end) ); + end +end +figure +subplot(1,2,1); +fill( xx1, yy1,red); +hold on +fill( xx2, yy2,blue); +xlim( [0,mx] ); ylim( [0,.5*mx] ); +text(.5,.07,'set C1'); +text(.5,.22,'set C2'); +plot( path(1,:), path(2,:),'ko-' ) +title('Path for Dykstra''s algo solving the intersection of 2 polygons'); + +subplot(1,2,2); +semilogy( errHist, 'o-' ); xlabel('iterations'); ylabel('error'); +title('Error for Dykstra''s method'); +set(gcf, 'Position', [400 200 800 400]); + +%% +% With Dykstra's algo (and alternating projections), we get +% very slow convergence, because of the very low angle between the +% two sets. Here is a zoom in on the graph of the iterates: +figure +fill( xx1, yy1,red); +hold on +fill( xx2, yy2,blue); +xlim( [0,mx] ); ylim( [0,.5*mx] ); +text(.5,.07,'set C1'); +text(.5,.22,'set C2'); +plot( path(1,:), path(2,:),'ko-' ) +title('Path for Dykstra''s algo solving the intersection of 2 polygons (zoom)'); +xlim( [.25,.4] ); +ylim( [.058,.1]) +%% Plot all the errors together +figure +semilogy( out.err(:,1),'-' ,'linewidth',3); +hold all +semilogy( errHist_1,'-','linewidth',3); +semilogy( errHist,'--','linewidth',3); +legend('TFOCS','Alternating projection','Dykstra'); +xlabel('iteration'); ylabel('error'); +%% Demo with two circles that intersect at (0,0) +center1 = [0;.5]; +radius1 = .5; +center2 = -center1; +radius2 = radius1; + +figure +rectangle('Position',[-radius1,0,2*radius1,2*radius1],'Curvature',[1,1],'FaceColor',red) +rectangle('Position',[-radius1,-2*radius2,2*radius1,2*radius1],'Curvature',[1,1],'FaceColor',blue) +axis equal +hold on +text(0,.5,'set C1'); +text(0,-.5,'set C2'); + + +offset1 = center1; +offset2 = center2; + +op1_d = prox_l2(radius1); +op2_d = prox_l2(radius2); +x0 = [1; .2]; + +% and some simpler operators that we will use with alternating projections +% and with Dykstra +proj1 = @(x) radius1*(x-center1)/norm(x-center1) + center1; +proj2 = @(x) radius2*(x-center2)/norm(x-center2) + center2; +%% solve in TFOCS +opts.maxIts = 300; +affineOperator = {eye(N),offset1;eye(N),offset2}; +dualProx = {op1_d,op2_d}; +% Solve in TFOCS: +[x,out,optsOut] = tfocs_SCD( [], affineOperator, dualProx , mu, x0, [], opts ); + +% Record the path: +path = recordPoints(); +path = [x0,path]; +% Plot: +figure +subplot(1,3,1); +rectangle('Position',[-radius1,0,2*radius1,2*radius1],'Curvature',[1,1],'FaceColor',red) +rectangle('Position',[-radius1,-2*radius2,2*radius1,2*radius1],'Curvature',[1,1],'FaceColor',blue) +hold on +text(0,.5,'set C1'); +text(0,-.5,'set C2'); +plot( path(1,:), path(2,:),'ko-','linewidth',2 ) +title('Path for TFOCS solving the intersection of 2 circles'); + +subplot(1,3,2); +rectangle('Position',[-radius1,0,2*radius1,2*radius1],'Curvature',[1,1],'FaceColor',red) +rectangle('Position',[-radius1,-2*radius2,2*radius1,2*radius1],'Curvature',[1,1],'FaceColor',blue) +hold on +plot( path(1,:), path(2,:),'ko-','linewidth',2 ) +xlim([0,.15]); +ylim([-.1,.1]); +title('(zoom)'); + +subplot(1,3,3); +semilogy( out.err,'o-') +title('Error of iterates for TFOCS method'); +set(gcf, 'Position', [400 200 800 400]); +%% solve via alternating projection method +maxIts = 300; +x = x0; +path = [x0]; +errHist = []; +for k = 1:maxIts + % basic alternating projection method: + x = proj1(x); + path= [path, x]; + x = proj2(x); + path= [path, x]; + errHist = [ errHist; err(x) ]; + if ~mod(k,50) + fprintf('Iter %4d, error is %.2e\n', k, errHist(end) ); + end +end +figure +subplot(1,3,1); +rectangle('Position',[-radius1,0,2*radius1,2*radius1],'Curvature',[1,1],'FaceColor',red) +rectangle('Position',[-radius1,-2*radius2,2*radius1,2*radius1],'Curvature',[1,1],'FaceColor',blue) +hold on +text(0,.5,'set C1'); +text(0,-.5,'set C2'); +plot( path(1,:), path(2,:),'ko-' ) +title('Path for alternating projection method solving the intersection of 2 circles'); + +subplot(1,3,2); +rectangle('Position',[-radius1,0,2*radius1,2*radius1],'Curvature',[1,1],'FaceColor',red) +rectangle('Position',[-radius1,-2*radius2,2*radius1,2*radius1],'Curvature',[1,1],'FaceColor',blue) +hold on +plot( path(1,:), path(2,:),'ko-','linewidth',2 ) +xlim([0,.15]); +ylim([-.1,.1]); +title('(zoom)'); + +subplot(1,3,3); +semilogy( errHist,'o-') +errHist_1 = errHist; +title('Error of iterates for alternating projection method'); +set(gcf, 'Position', [400 200 800 400]); +%% solve via Dykstra's algorithm + +maxIts = 300; +x = x0; +[p,q] = deal( 0*x0 ); +path = [x0]; +errHist = []; +for k = 1:maxIts + % If x + p is feasible, the y = (x+p), so p=x+p-y = 0. + y = proj1( x + p ); + p = x + p - y; + x = proj2( y + q ); + q = y + q - x; + path = [path,y]; + path = [path,x]; + errHist = [ errHist; err(x) ]; + if ~mod(k,50) + fprintf('Iter %4d, error is %.2e\n', k, errHist(end) ); + end +end +figure +subplot(1,3,1); +rectangle('Position',[-radius1,0,2*radius1,2*radius1],'Curvature',[1,1],'FaceColor',red) +rectangle('Position',[-radius1,-2*radius2,2*radius1,2*radius1],'Curvature',[1,1],'FaceColor',blue) +hold on +text(0,.5,'set C1'); +text(0,-.5,'set C2'); +plot( path(1,:), path(2,:),'ko-' ) +title('Path for Dykstra''s algo solving the intersection of 2 circles'); + +subplot(1,3,2); +rectangle('Position',[-radius1,0,2*radius1,2*radius1],'Curvature',[1,1],'FaceColor',red) +rectangle('Position',[-radius1,-2*radius2,2*radius1,2*radius1],'Curvature',[1,1],'FaceColor',blue) +hold on +plot( path(1,:), path(2,:),'ko-','linewidth',2 ) +xlim([0,.15]); +ylim([-.1,.1]); +title('(zoom)'); + +subplot(1,3,3); +semilogy( errHist,'o-') +title('Error of iterates for Dykstra''s algo'); +set(gcf, 'Position', [400 200 800 400]); + +%% Plot all the errors together +figure +semilogy( out.err(:,1),'-' ,'linewidth',3); +hold all +semilogy( errHist_1,'-','linewidth',3); +semilogy( errHist,'--','linewidth',3); +legend('TFOCS','Alternating projection','Dykstra'); +xlabel('iteration'); ylabel('error'); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/examples/demos/project2DCone.m b/examples/demos/project2DCone.m new file mode 100644 index 0000000..6e74b4e --- /dev/null +++ b/examples/demos/project2DCone.m @@ -0,0 +1,74 @@ +function op = project2DCone(x1, x2 ) +% OP = project2DCone( y, x1, x2 ) +% represents the 2D point cone defined +% by the vectors x1 and x2. x1 should be "left" of x2, +% and the cone is defined as the region to the right +% of x1 and to the left of x2. + +error(nargchk(1,2,nargin)); +if nargin < 2, u = []; end +%op = @proj_Rplus_impl; +% bugfix, March 2011: +op = @(varargin)proj_cone2(x1,x2,varargin{:}); + +function [v,x] = proj_cone2(x1,x2,y,t) +% "t" doesn't matter, since the only values are 0 and Inf +v = 0; +if size(y,2) > 1, y = y.'; end + +% Assume that "x1" is active +n1 = [x1(2); -x1(1)]; % the normal direction +t1 = -y'*n1/(n1'*n1); + + +% Now assume "x2" is active: +n2 = [x2(2); -x2(1)]; +t2 = -y'*n2/(n2'*n2); + +switch nargin, + case 3, + if nargout == 2, + error( 'This function is not differentiable.' ); + end + if ( t1 > 0 ) && ( t2 < 0 ) + v = 0; + else + v = Inf; + end + case 4, + if y'*x1 <= 0 && y'*x2 <= 0 + x = 0*y; v = 0; + return; + end + + p1 = y + t1*n1; + p2 = y + t2*n2; + + % now determine which one is active: + if p1'*x1 < 0 + % p1 cannot be active + x = p2; + elseif p2'*x2 < 0 + % p2 cannot be active + x = p1; + elseif ( t1 > 0 ) && ( t2 < 0 ) + % we are feasible! + x = y; + else + % both could be active, so pick the best: + if norm( p1 - y ) < norm( p2 - y ) + x = p1; + else + x = p2; + end + end + + + otherwise, + error( 'Wrong number of arguments.' ); +end + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/demos/recordPoints.m b/examples/demos/recordPoints.m new file mode 100644 index 0000000..8bea9a4 --- /dev/null +++ b/examples/demos/recordPoints.m @@ -0,0 +1,17 @@ +function er = recordPoints( x ) + +persistent history + +if nargin == 0 + er = history; + history = []; + return; +end + +history = [history, x ]; +er = 0; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/examples/largescale/BugsBunny.mat b/examples/largescale/BugsBunny.mat new file mode 100644 index 0000000..c56305b Binary files /dev/null and b/examples/largescale/BugsBunny.mat differ diff --git a/examples/largescale/PsiTransposeWFF.m b/examples/largescale/PsiTransposeWFF.m new file mode 100644 index 0000000..644e610 --- /dev/null +++ b/examples/largescale/PsiTransposeWFF.m @@ -0,0 +1,58 @@ +function y = PsiTransposeWFF(x,w_type,log_length,min_scale,max_scale,shift_redundancy,freq_redundancy, PLOT) +% Windowed Fourier Frame Analysis +% y = PsiTransposeWFF( x, w_type, log_length, min_scale, max_scale,... +% shift_redundancy, freq_redundancy, plot ) +% w_type is the type of window. Currently, this supports 'isine' (iterate sine) +% and 'gaussian'. Default: 'isine' (to use default, set this to [] ). +% +% Core code written by Peter Stobbe; modifications by Stephen Becker +if isempty(w_type), w_type = 'isine'; end +if nargin < 8 || isempty(PLOT), PLOT = false; end + +% w is a is a vector of with the window of the largest scale +% smaller scale windows are just this subsampled +[w, window_redundancy] = make_window(max_scale,w_type); + +y = []; +c = ((max_scale - min_scale + 1)*window_redundancy*2.^((1:max_scale)'+freq_redundancy+shift_redundancy)).^-.5; + + +for k = min_scale:max_scale + M = 2^(log_length-k) +(2^(log_length-k)+1)*(2^shift_redundancy-1); + z = [myRepMat(x,2^shift_redundancy); zeros(2^k - 2^(k-shift_redundancy),2^shift_redundancy)]; + z = reshape(z,2^k,M); + z = z.*myRepMat(w(2^(max_scale-k)*(1:2^k)'),M); + z(2^(k+freq_redundancy),M) = 0; + z = fft(z); + z = [z(1,:)*c(k); real(z(2:end/2,:))*c(k-1); ... + z(end/2+1,:)*c(k); imag(z(end/2+2:end,:))*c(k-1)]; + y = [y; z(:)]; +end + +function B = myRepMat(A,n) +B = A(:,ones(n,1)); + +function [w,window_redundancy] = make_window(max_scale,w_type) +% [w,window_redundancy] = make_window(max_scale,w_type) +% w_type can be +% 'isine' for iterated sine +% 'gaussian' for gaussian +% 'trapezoid' for trapezoidal shape (not a frame) + +x = (1:2^max_scale)'/2^(max_scale-1)-1; +if isequal(w_type,'isine') + w = sin(pi/4*(1+cos(pi*x))); + window_redundancy = 1/2; +elseif isequal(w_type,'gaussian') + w = exp(-x.^2*8*log(2)); + window_redundancy = mean(w.^2); +elseif isequal(w_type,'trapezoid') + w = min(1,2*(1-abs(x))); + window_redundancy = mean(w.^2); +else + disp('Error in make_window: unknown window type'); + disp('Options are: isine, gaussian, trapezoid'); +end +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/largescale/PsiWFF.m b/examples/largescale/PsiWFF.m new file mode 100644 index 0000000..38b9877 --- /dev/null +++ b/examples/largescale/PsiWFF.m @@ -0,0 +1,62 @@ +function x = PsiWFF(y,w_type,log_length,min_scale,max_scale,shift_redundancy,freq_redundancy); +% Windowed Fourier Frame Synthesis +% x = PsiWFF(y,w_type,log_length,min_scale,max_scale,shift_redundancy,freq_redundancy); +% w_type is the type of window. Currently, this supports 'isine' (iterate sine) +% and 'gaussian'. Default: 'isine' (to use default, set this to [] ). +% written by Peter Stobbe, modified by Stephen Becker + +if isempty(w_type), w_type = 'isine'; end + +% w is a is a vector of with the window of the largest scale +% smaller scale windows are just this subsampled +[w, window_redundancy] = make_window(max_scale,w_type); + +x = zeros(2^log_length,1); +c = ((max_scale - min_scale + 1)*window_redundancy*2.^((1:max_scale)'+freq_redundancy+shift_redundancy)).^-.5; +cur = 0; + + +for k = min_scale:max_scale + M = 2^(log_length-k) +(2^(log_length-k)+1)*(2^shift_redundancy-1); + N = 2^(k+freq_redundancy); + z = reshape(y(cur + (1:N*M)),N,M); + cur = cur + N*M; + z = [z(1,:)*c(k); z(2:N/2,:)*c(k-1);... + z(N/2+1,:)*c(k); -1i*z(N/2+2:N,:)*c(k-1)]; + z = real(fft(z)); + z = z(1:2^k,:).*myRepMat(w(2^(max_scale-k)*(1:2^k)),M); + z = reshape(z,[],2^shift_redundancy); + x = x + sum(z(1:2^log_length,:),2); +end + + +%--------------- + +function B = myRepMat(A,n) +B = A(:,ones(n,1)); + +function [w,window_redundancy] = make_window(max_scale,w_type) +% [w,window_redundancy] = make_window(max_scale,w_type) +% w_type can be +% 'isine' for iterated sine +% 'gaussian' for gaussian +% 'trapezoid' for trapezoidal shape (not a frame) + +x = (1:2^max_scale)'/2^(max_scale-1)-1; +if isequal(w_type,'isine') + w = sin(pi/4*(1+cos(pi*x))); + window_redundancy = 1/2; +elseif isequal(w_type,'gaussian') + w = exp(-x.^2*8*log(2)); + window_redundancy = mean(w.^2); +elseif isequal(w_type,'trapezoid') + w = min(1,2*(1-abs(x))); + window_redundancy = mean(w.^2); +else + disp('Error in make_window: unknown window type'); + disp('Options are: isine, gaussian, trapezoid'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/largescale/README.txt b/examples/largescale/README.txt new file mode 100644 index 0000000..804dde7 --- /dev/null +++ b/examples/largescale/README.txt @@ -0,0 +1,3 @@ +This directory contains interesting applications, + +e.g. denoising an image diff --git a/examples/largescale/example_quantumTomography.m b/examples/largescale/example_quantumTomography.m new file mode 100644 index 0000000..b579f70 --- /dev/null +++ b/examples/largescale/example_quantumTomography.m @@ -0,0 +1,223 @@ +%{ + 5/28/09, 10/28/10 + pauli-matrix tensor measurements + Stephen Becker, srbecker@caltech.edu + + See http://arxiv.org/abs/0909.3304 + +We seek to find a low-rank matrix X, because X represents the quantum state +of d atoms (X has dimensions n x n where n = 2^d, due to quantum entanglement). +X is Hermitian and positive semi-definite, and furthermore trace(X) = 1. + +From experiments, we have measurements b_i = < A_i, X > where <,> is the +trace inner product. A_i is a Hermitian matrix, and is a tensor product +of Pauli matrices. The observations are noisy. + +To recover X, we can consider solving several related variants: + +notation: +x = vec(X), X = mat(x) + + + minimize 1/2||A*x - b ||^2 + subject to + mat(x) >= 0 + trace(mat(x)) <= 1 + +(this variant does not require smoothing) + +or + + minimize trace(mat(x)) + subject to + mat(x) >= 0 + ||A*x - b || <= eps +(this variant requires smoothing and/or continuation ) + +%} +%% Setup a matrix +randn('state',2009); +rand('state',2009); + +% -- Generate the state matrix "M" (aka rho) +n1 = 128; % fast +% n1 = 256; +% n1 = 512; % works, but slow +n2 = n1; r = 2; +M = ( randn(n1,r) + 1i*randn(n1,r) )*... + ( randn(r,n2) + 1i*randn(r,n2) )/2; +M=M*M'; % M is Pos. Semidefinite +M = M / trace(M); % and has trace 1 + +df = r*(n1+n2-r); % The degrees of freedom +oversampling = 5; % Information-theoretic limit corresponds + % to oversampling = 1 +m = min(5*df,round(.99*n1*n2) ); + +fprintf('%d x %d matrix, %d measurements (%.1f%%, or %.1f oversampling)\n',... + n1,n2,m,100*m/(n1*n2),m/df ); +fprintf('True rank is %d\n', r); +if ~isreal(M), fprintf('Matrix is complex\n'); end + +vec = @(x) x(:); +mat = @(x) reshape(x,n1,n2); + +%% make some pauli matrices, to use for observations +%{ +Using the convention: + PX = [0,1;1,0]; PY = [0,-1i;1i,0]; PZ =[1,0;0,-1]; PI = eye(2); + X will be labelled "1", Y labeled "2", Z "3" and I "4" +%} +myRandint = @(M,N,d) ceil(d*rand(M,N)); + +PAULI_T = sparse([],[],[],n1*n2,m,m*n1 ); +fprintf('Taking measurements... '); +for i=1:m + fprintf('\b\b\b\b\b\b%5.1f%%', 100*i/m ); + % pick from the X, Y, Z and I uniformly at random + list = myRandint( log2(n1),1,4 ); + E_i = explicitPauliTensor( list ); + % NOTE: with this definition, E_i is Hermitian, + % but not positive semidefinite, and not scaled properly. + + PAULI_T(:,i) = E_i(:); % access via column is MUCH faster +end +fprintf('\n'); +PAULI = PAULI_T.'; % transpose it, since we were implicitly +clear PAULI_T % dealing with transpose earlier + % (since updating the columns of a sparse + % matrix is much more efficient than updating + % the rows ) + +fprintf('Computing the scaling factor...'); +% scale = 1/normest(PAULI*PAULI',1e-2); +% scale = 1/my_normest( @(x) PAULI*(PAULI'*x), size(PAULI,1)*[1,1] ); +fprintf(' done.\n'); + +data = PAULI*vec(M); +% even if measurements are complex, data should still be real +fprintf('due to roundoff, measurements have %.2e imaginary part\n',... + norm(imag(data))/norm(data) ); +data = real(data); + +%% Add some noise +%{ + Physics of noise: + in addition to noise from measurement error, we have a dominant + source of 'noise' which is because the outcome of our + measurements is really the result of measuring a quantum wave + function. Experimentalists will repeat the same + measurement many times, so they can estimate the true state, + but there is inherent quantum mechanical uncertainty to this. + Basically, we have a state that is determined by a Bernoulli + parameter 0 < p < 1, but the measurement returns either 0 or 1, + so we have to take a lot of measurements to estimate p. + + There's also depolarizing noise, which we don't discuss here. + + For simplicity in this example, we don't worry about that model, + and just add in some pseudo-random white noise +%} + +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); +snr = 40; +b = myAwgn( data, snr ); +rms = @(x) norm(x)/sqrt(length(x)); +snrF = @(sig,noise) 20*log10( rms(sig)/rms(noise) ); +sigma = std( b - data ); +fprintf('SNR of measurements is %.1f dB, and std of noise is %.2f\n',... + snrF(data,data-b), sigma ); +%% Solve in TFOCS +%{ + +Formulation: + minimize 1/2||PAULI*x - b ||^2 + subject to + mat(x) >= 0 + trace(mat(x)) <= 1 + +No smoothing is required, but... it's hard to exploit the low-rank +structure of the problem + +We do not expect the "error" to go to zero, since the data are noisy +and we don't have the true answer, so we're not really looking +at "error" of the software, but rather modelling error... + +%} +opts = []; +opts.maxIts = 300; +opts.printEvery = 25; +if n1 > 256 + opts.maxIts = 50; + opts.printEvery = 5; +end +opts.errFcn = {@(f,x) norm( vec(x) - vec(M) )/norm(M,'fro')}; +% this may be slow: +% opts.errFcn{2} = @(f,x) rank(mat(x)); + +tau = 1; % constraint on trace +x0 = []; + +A = linop_matrix( PAULI, 'C2R' ); +[x,out,optsOut] = tfocs(smooth_quad,{A,-b}, proj_psdUTrace(tau) , x0, opts ); +X = mat(x); + +figure(1); +semilogy( out.err(:,1) ); +hold all +%% Solve in TFOCS +%{ + +Formulation: + minimize trace(mat(x)) + mu/2*||x-x0||^2 + subject to + mat(x) >= 0 + ||PAULI*x - b || <= eps + +We solve this via a dual method, hence the mu term. +We can reduce its effect by using continuation (i.e. updating x0) + +%} +opts = []; +opts.maxIts = 50; +opts.printEvery = 25; +opts.errFcn = {@(f,dual,x) norm( vec(x) - vec(M) )/norm(M,'fro')}; +% this may be slow: +% opts.errFcn{2} = @(f,dual,x) rank(mat(x)); +opts.stopCrit = 4; +opts.tol = 1e-8; +contOpts = []; +contOpts.betaTol = 10; +contOpts.maxIts = 5; + +epsilon = norm(b-data); % cheating a bit... +x0 = []; +z0 = []; +mu = 5; + +largescale = ( n1*n2 >= 256^2 ); +% obj = prox_nuclear(1,largescale); % works, but unnecessarily general +obj = prox_trace(1,largescale); + +prox = prox_l2(epsilon); +A = linop_matrix( PAULI, 'C2R' ); +I = []; +It = linop_handles({ [n1,n2],[n1*n2,1] }, vec, mat, 'C2C' ); +A = linop_compose( A, It ); + +[x,out,opts] = tfocs_SCD( obj,{ A, -b; I, 0 }, {prox, proj_psd}, mu, x0, z0, opts, contOpts ); +X = mat(x); + +fprintf('Error is %.2e, error after renormalizing is %.2e\n',... + opts.errFcn{1}(1,1,X), opts.errFcn{1}(1,1,X/trace(X) ) ); +fprintf('Rank of X is %d\n', rank(X) ); + +figure(1); +semilogy( out.err(:,1) ); +hold all +semilogy(out.contLocations, out.err(out.contLocations,1), 'o' ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/largescale/explicitPauliTensor.m b/examples/largescale/explicitPauliTensor.m new file mode 100644 index 0000000..8e145ad --- /dev/null +++ b/examples/largescale/explicitPauliTensor.m @@ -0,0 +1,27 @@ +function u = explicitPauliTensor( list ) +% u = explicitPauliTensor( list ) +% makes the explicit matrix corresponding to the tensor +% product of Pauli matrices. +% "list" should be an ordered vector, where each entry is +% either 1, 2, 3 or 4, correspodning to the X, Y, Z and I +% Pauli matrices, respectively. + +if ~isvector(list) || max(list) > 4 || min(list) < 1 + error('Error making Pauli matrices'); +end + +PX = [0,1;1,0]; PY = [0,-1i;1i,0]; PZ =[1,0;0,-1]; PI = eye(2); + +PAULI{1} = PX; +PAULI{2} = PY; +PAULI{3} = PZ; +PAULI{4} = PI; + +u=1; +for i = 1:length(list) + u = kron( u, sparse( PAULI{list(i)} )); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/largescale/findWeights.m b/examples/largescale/findWeights.m new file mode 100644 index 0000000..9a8d995 --- /dev/null +++ b/examples/largescale/findWeights.m @@ -0,0 +1,26 @@ +function w = findWeights( coeff, p ) +% weights = findWeights( coefficients, p ) +% creates a nice vector of reweighting coefficients +% for use with re-weighted l1. +% The regularization parameter is chosen to be the coefficient +% that accounts for p-percent (0 1 + error('p must be a scalar between [0,1]' ); +end + +coeff = abs(coeff); +coeffSort = sort(coeff,'descend'); +c = cumsum( coeffSort.^2 ); +indx = find( c/c(end) > p, 1, 'first' ); +if ~isempty(indx) + delta = coeffSort( indx ); +else + delta = 1e-15; +end + +w = delta./( coeff + delta ); +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/largescale/image_denoising_withSPOT.m b/examples/largescale/image_denoising_withSPOT.m new file mode 100644 index 0000000..a8ce478 --- /dev/null +++ b/examples/largescale/image_denoising_withSPOT.m @@ -0,0 +1,284 @@ +%{ + Tests total-variation problem and l1 analysis + on a large-scale example + + min_x alpha*||x||_TV + beta*||Wx||_1 +s.t. + || A(x) - b || <= eps + +where W is a wavelet operator. + + +This requires: + + (1) SPOT www.cs.ubc.ca/labs/scl/spot/ + (2) Wavelab www-stat.stanford.edu/~wavelab/ +and optionally, + (3) CurveLab www.curvelet.org/ + + Also demonstrates how to use SPOT with TFOCS. + To install SPOT, please visit: + http://www.cs.ubc.ca/labs/scl/spot/ + + We use SPOT to call WaveLab and CurveLab, + which may need to be installed separately. + Please contact the TFOCS and/or SPOT authors if you need help. + + Also, SPOT v1.0 has a typo in its curvelet code: + after adding SPOT to your path, edit this file + edit spotbox-1.0p/opCurvelet + and change all instances of "fdct_c2v" and "fdct_v2c" + to "spot.utils.fdct_c2v" and "spot.utils.fdct_v2c", resp. + +%} + +% Setting up the various packages (change this for your computer): +addpath ~/Dropbox/TFOCS/ +addpath ~/Documents/MATLAB/spotbox-1.0p/ +addpath ~/Documents/MATLAB/CurveLab-2.1.2/fdct_wrapping_cpp/mex/ + +myAwgn = @(x,snr) x +10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*... + randn(size(x)); +%% Load an image and take noisy measurements + + +n1 = 256; +n2 = 256; +N = n1*n2; +x = phantom(n1); + +% Signal-to-noise ratio, in dB +snr = 5; + + +x_original = x; +mat = @(x) reshape(x,n1,n2); +% vec = @(x) x(:); + +% Add noise: +randn('state',245); rand('state',245); +x_noisy = myAwgn( x_original, snr); + +maxI = max(vec(x_original)); % max pixel value +PSNR = @(x) 20*log10(maxI*sqrt(N)/norm(vec(x)-vec(x_original) ) ); + +% Take measurements +b = vec(x_noisy); +b_original = vec(x_original); +EPS = .8*norm(b-b_original); + +figure(2); imshow( [x_original, x_noisy] ); drawnow; +M=N; +fprintf('Denoising problem, %d x %d, signal has SNR %d dB\n',M, N, round(snr) ); + +REWEIGHT = false; % whether to do one iteration of reweighting or not +%% Call the TFOCS solver + +mu = 5; +er = @(x) norm(x(:)-x_original(:))/norm(x_original(:)); +opts = []; +opts.errFcn = @(f,dual,primal) er(primal); +opts.maxIts = 100; +opts.printEvery = 20; +opts.tol = 1e-4; +opts.stopcrit = 4; + +x0 = x_noisy; +z0 = []; % we don't have a good guess for the dual + +% build operators: +A = linop_handles([N,N], @(x)x, @(x) x ); +normA2 = 1; +W_wavelet = linop_spot( opWavelet(n1,n2,'Daubechies') ); +if exist('fdct_wrapping') + DO_CURVELETS = true; + W_curvelet = linop_spot( opCurvelet(n1,n2) ); +else + % You either don't have curvelab installed + % or you need to add it to the matlab path. + DO_CURVELETS = false; +end +W_tv = linop_TV( [n1,n2] ); +normWavelet = linop_normest( W_wavelet ); +if DO_CURVELETS, normCurvelet = linop_normest( W_curvelet ); end +normTV = linop_TV( [n1,n2], [], 'norm' ); + +contOpts = []; +contOpts.maxIts = 4; + +%% -- First, solve just via wavelet -- +clc; disp('WAVELETS'); +[x_wavelets,out_wave] = solver_sBPDN_W( A, W_wavelet, b, EPS, mu, ... + x0(:), z0, opts, contOpts); + +if REWEIGHT + % do some re-weighting: + coeff = W_wavelet( x_wavelets, 1 ); + weights = findWeights( coeff, .85 ); + W_weights = linop_handles( [length(weights),length(weights)], @(x)weights.*x,... + @(y) weights.*y, 'R2R' ); + [x_wavelets,out_wave] = solver_sBPDN_W( A, ... + linop_compose(W_weights,W_wavelet), ... + b, EPS, mu, x_wavelets, out_wave.dual, opts, contOpts); + % and of course, you could keep iterating... +end +%% -- Second, solve just via curvelets -- +if DO_CURVELETS + opts_copy = opts; + opts_copy.maxIts = 10; + opts_copy.normA2 = 1; + opts_copy.normW2 = normCurvelet^2; + clc; disp('CURVELETS'); + [x_curvelets,out_curve] = solver_sBPDN_W( A, W_curvelet, b, EPS, mu, x0(:), z0, opts_copy); + + if REWEIGHT + % do some re-weighting: + coeff = W_curvelet( x_curvelets, 1 ); + weights = findWeights( coeff, .85 ); + W_weights = linop_handles( [length(weights),length(weights)], @(x)weights.*x,... + @(y) weights.*y, 'R2R' ); + % really, we should update the norm of W, since it is changing slightly + % by adding in the weights, but that is a slow calculation, and the norm + % doesn't change too much, so we skip that. + [x_curvelets,out_curve] = solver_sBPDN_W( A, ... + linop_compose(W_weights,W_curvelet), ... + b, EPS, mu, x_curvelets, out_curve.dual, opts_copy); + end +end + +%% -- Third, solve just via TV -- +clc; disp('TV'); +opts_copy = opts; +opts_copy.normA2 = 1; +opts_copy.normW2 = normTV^2; +opts_copy.continuation = false; +[x_tv,out_tv] = solver_sBPDN_W( A, W_tv, b, EPS, mu, x0(:), z0, opts_copy, contOpts); + +if REWEIGHT + % do some re-weighting: + coeff = W_tv( x_tv, 1 ); + weights = findWeights( coeff, .95 ); + W_weights = linop_handles( [length(weights),length(weights)], @(x)weights.*x,... + @(y) weights.*y, 'C2C' ); + % really, we should update the norm of W, since it is changing slightly + % by adding in the weights, but that is a slow calculation, and the norm + % doesn't change too much, so we skip that. + [x_tv,out_tv] = solver_sBPDN_W( A, ... + linop_compose(W_weights,W_tv), ... + b, EPS, mu, x_tv, out_tv.dual, opts_copy); +end +%% -- Fourth, combine wavelets (or curvelets) and tv -- +alpha = 1; % weight of TV term +beta = .05; % weight of wavelet term +normW12 = normTV^2; +if DO_CURVELETS, normW22 = normCurvelet^2; beta = .5; +else, normW22 = normWavelet^2; end + +% x =solver_sBPDN_WW( A, alpha, W_tv, beta, ... +% W_wavelet, b, EPS, mu,vec(x0), z0, opts, contOpts); + +% this is what solver_sBPDN_WW is doing: +W1 = W_tv; +if DO_CURVELETS + W2 = W_curvelet; +else + W2 = W_wavelet; +end +prox = { prox_l2( EPS ), ... + proj_linf( alpha ),... + proj_linf( beta ) }; +affine = { A, -b; W1, 0; W2, 0 }; + +% x = tfocs_SCD( [], affine, prox, mu, x0(:), z0, opts, contOpts ); +% X_wavelets_tv = mat(x); +%% try with box constraints +% Constrain the image pixels to be in the range 0 <= x <= 1 +prox = { prox_l2( EPS ), ... + proj_linf( alpha ),... + proj_linf( beta ), ... + proj_Rplus , ... + proj_Rplus}; + +affine = { A, -b; W1, 0; W2, 0 ; 1, 0; -speye(n1*n2), maxI*ones(n1*n2,1) }; + +x_wavelets_tv = tfocs_SCD( [], affine, prox, mu, x0(:), z0, opts, contOpts ); +fprintf('Min and max entry of recovered image are %.1f and %.1f\n', min(min(x_wavelets_tv)),... + max(max(x_wavelets_tv)) ); +%% another way to use box constraints +% Instead of thinking of it as two scaled X >= 0 constraints, +% we call the special purpose atom + +prox = { prox_l2( EPS ), ... + proj_linf( alpha ),... + proj_linf( beta ), ... + prox_boxDual(0,maxI,-1) }; + +affine = { A, -b; W1, 0; W2, 0 ; 1, 0 }; +opts_copy = opts; +opts_copy.continuation = false; +opts_copy.maxIts = 30; +[x_wavelets_tv,out] = tfocs_SCD( [], affine, prox, mu, x0(:), z0, opts_copy, contOpts ); + +if REWEIGHT + % do some re-weighting: + weights = findWeights( W1(x_wavelets_tv,1), .95 ); + W_weights1 = linop_handles( [length(weights),length(weights)], @(x)weights.*x,... + @(y) weights.*y, 'C2C' ); + weights = findWeights( W2(x_wavelets_tv,1), .95 ); + W_weights2 = linop_handles( [length(weights),length(weights)], @(x)weights.*x,... + @(y) weights.*y, 'C2C' ); + + affine = { A, -b; linop_compose(W_weights1,W1), 0; ... + linop_compose(W_weights2,W2) , 0 ; -1, 0 }; % Note: we need the "-1", not +1 + [x_wavelets_tv,out] = tfocs_SCD( [], affine, prox, mu, x0(:), out.dual, opts_copy, contOpts ); + +end +fprintf('Min and max entry of recovered image are %.1f and %.1f\n', min(min(x_wavelets_tv)),... + max(max(x_wavelets_tv)) ); +%% Plot everything +figure(1); clf; +splot = @(n) subplot(2,3,n); + +splot(1); +imshow(x_original); +title(sprintf('Noiseless image,\nPSNR %.1f dB', Inf )); + +splot(2); +imshow(x_noisy); +title(sprintf('Noisy image,\nPSNR %.1f dB', PSNR(x_noisy) )); + +if DO_CURVELETS + splot(3); + imshow(mat(x_curvelets)); + title(sprintf('Curvelet regularization,\nPSNR %.1f dB', PSNR(x_curvelets) )); +end + +splot(4); +imshow(mat(x_wavelets)); +title(sprintf('Wavelet regularization,\nPSNR %.1f dB', PSNR(x_wavelets) )); + +splot(5); +imshow(mat(x_tv)); +title(sprintf('TV regularization,\nPSNR %.1f dB', PSNR(x_tv) )); + +splot(6); +imshow(mat(x_wavelets_tv)); +if DO_CURVELETS + title(sprintf('Curvelets + TV, and box constraints,\nPSNR %.1f dB', PSNR(x_wavelets_tv) )); +else + title(sprintf('Wavelets + TV, and box constraints,\nPSNR %.1f dB', PSNR(x_wavelets_tv) )); +end + +%% Bonus: movie. +% This uses the helper file "plotNow.m" to show you the image +% at every iteration. It slows down the algorithm on purpose +% so that you have time to watch it. +figure(2); clf; plotNow(); +time_delay = 0.1; +opts.errFcn = {@(f,d,p) er(p), @(f,d,p) plotNow( mat(p), time_delay) }; +x_tv = solver_sBPDN_W( A, W_tv, b, EPS, .1*mu, vec(x0), z0, opts); +title('THAT''S ALL FOLKS'); +%% +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/largescale/plotNow.m b/examples/largescale/plotNow.m new file mode 100644 index 0000000..0ab7071 --- /dev/null +++ b/examples/largescale/plotNow.m @@ -0,0 +1,40 @@ +function out = plotNow( x, pause_t , plotFcn ) +% out = plotNow( x, pause_t ) +% helper function for plotting a movie during +% the TFOCS iterations. +% Call this function with no inputs in order +% to reset it. +% Call this with a matrix and it will plot it using "imshow". +% +% The input "pause_t" controls how long to pause +% (this is useful when solving a small problem that would +% otherwise progress too fast between iterates). +% +% out = plotNow( x, pause_t, plotFcn ) +% will plot "x" using "plotFcn" instead of the default "imshow" + +persistent counter +if isempty(counter) + counter = 0; +end +if nargin == 0 + counter = 0; + return; +end +counter = counter + 1; + +if nargin < 3 || isempty(plotFcn), plotFcn = @(x) imshow(x); end + +plotFcn(x); +title(sprintf('Iteration %2d',counter)); +drawnow + +if nargin > 1 && ~isempty(pause_t) + pause(pause_t); +end + +out = 0; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/largescale/test_SVM.m b/examples/largescale/test_SVM.m new file mode 100644 index 0000000..b03424c --- /dev/null +++ b/examples/largescale/test_SVM.m @@ -0,0 +1,171 @@ +%{ + Support Vector Machine (SVM) example + + This is not a largescale test but it's neat, so it's in this directory + + We have binary data, and the two classes are labeled +1 and -1 + The data is d-dimensional, and we have n samples + + + This example show show to solve the standard SVM using the hinge-loss + and l2 penalty. Then it shows how it is easy with TFOCS to generalize + the SVM problem and use the l1 penalty instead of the l2 penalty, + which has the effect of encouraging sparsity. + +%} + +% Before running this, please add the TFOCS base directory to your path + + +% Generate a new problem + +randn('state',23432); +rand('state',3454); + +n = 30; +d = 2; % 2D data is nice because it's easy to visualize + +n1 = round(.5*n); % number of -1 (this is data from one "class") +n2 = n - n1; % number of +1 (this is data from the other "class") + +% Generate data +mean1 = [-.5,1]; +% mean2 = [.6,.2]; % can be separated +mean2 = [.1,.5]; % cannot be separated, so a bit more interesting +s = .25; % standard deviation +x1 = repmat(mean1,n1,1) + s*randn(n1,2); +x2 = repmat(mean2,n2,1) + s*randn(n2,2); + +X = [x1; x2]; +labels = [ ones(n1,1); -ones(n2,1) ]; + +figure(1); clf; +hh = {}; +hh{1}=plot(x1(:,1), x1(:,2), 'o' ); +hold all +hh{2}=plot(x2(:,1), x2(:,2), '*' ); +legend('Class 1','Class 2'); +xl = get(gca,'xlim'); +yl = get(gca,'ylim'); + +%% compute a separating hyperplane with SVM +% First, we use CVX to get a reference solution + +hinge = @(x) sum(max(0,1-x)); +lambda_1 = 10; +lambda_2 = 4; + +if exist( 'cvx_begin','file') + +% -- Problem 1: standard SVM in primal form +PROBLEM = 1; +cvx_begin + cvx_quiet true + cvx_precision best + variables a(d) b(1) + minimize hinge( labels.*( X*a - b ) ) + lambda_1*norm(a,2) +cvx_end +SLOPE{PROBLEM} = a; +INTERCEPT{PROBLEM} = b; + +% -- Problem 2: "sparse" SVM (use l1 norm as penalty) +PROBLEM = 2; +cvx_begin + cvx_quiet true + cvx_precision best + variables a(d) b(1) + minimize hinge( labels.*( X*a - b ) ) + lambda_2*norm(a,1) +cvx_end +SLOPE{PROBLEM} = a; +INTERCEPT{PROBLEM} = b; + +% An equivalent way to solve via method 2: +% X_aug = [diag(labels)*X, -labels ]; % include the "b" variable +% cvx_begin +% variables a(d+1) +% minimize hinge( X_aug*a ) + lambda_2*norm(a(1:2),1) +% cvx_end + +else + % load the pre-computed reference solutions + lambda_1 = 10; + lambda_2 = 4; + SLOPE = { [ -1.084604434515675; 0.783281331181009]; ... + [ -2.391008972722243; 0] }; + INTERCEPT = { [0.75708813868169] ; [0.260615633092470] }; +end + +%% plot the separating hyper-planes +for PROBLEM = 1:length(SLOPE) + a = SLOPE{PROBLEM}; b = INTERCEPT{PROBLEM}; + + if abs(a(2)/a(1)) < 1e-10 + % vertical line: + hh{PROBLEM+2}=line( [1,1]*b/a(1), [-.3,2] ); + else + grid = linspace(-.5,1,100); + hh{PROBLEM+2}=plot( grid, (b-a(1)*grid)/a(2) ); + end +end +legend([hh{:}],'Class 1','Class2','SVM','sparse SVM'); +xlim(xl); ylim(yl); + +%% Solve Problem 2 in TFOCS +PROBLEM = 2; +linearF = diag(labels)*[ X, -ones(n,1) ]; +objF = prox_l1(lambda_2*[1,1,0]'); +% objF = prox_l1(lambda_2); +prox = prox_hingeDual(1,1,-1); % but use -x since we want polar cone, not dual cone + +mu = 1e-2; +opts = []; +opts.maxIts = 1500; +opts.continuation = false; +opts.stopCrit = 4; +opts.tol = 1e-12; +opts.errFcn = @(f,d,x) norm( x - [SLOPE{PROBLEM};INTERCEPT{PROBLEM}] ); +x0 = zeros(d+1,1); % i.e. 3 for 2D SVM +y0 = zeros(n,1); +[ak,out,optsOut] = tfocs_SCD( objF,linearF, prox, mu,x0,y0,opts); +% an alternative way to call it: +% [ak,out,optsOut] = tfocs_SCD( objF,{linearF,[]}, prox, mu,x0,y0,opts); + +%% Solve Problem 2 in TFOCS a different way +% We dualize the l1 term now +PROBLEM = 2; +prox2 = { prox, proj_linf(lambda_2) }; +linearF2 = diag( [ones(d,1);0] ); +opts.errFcn = @(f,d,x) norm( x - [SLOPE{PROBLEM};INTERCEPT{PROBLEM}] ); +[ak,out,optsOut] = tfocs_SCD([],{linearF,[];linearF2,[]}, prox2, mu,[],[],opts); + + +%% Solve Problem 1 in TFOCS +% Keep l1 term in primal; this is an "experimental" feature +% since the l1 proximity operator is not exactly in closed form +% (it relies on a 1D linesearch; this should be very fast, but +% it could have numerical issues for very high accuract solutions) +PROBLEM = 1; +objF = prox_l2(lambda_1*[1,1,0]'); +opts.errFcn = @(f,d,x) norm( x - [SLOPE{PROBLEM};INTERCEPT{PROBLEM}] ); +% opts.tol = 1e-16; +opts.continuation = true; +mu2 = 10*mu; +[ak,out,optsOut] = tfocs_SCD( objF,linearF, prox, mu2,[],[],opts); + +%% Solve Problem 1 in TFOCS a different way +% Dualize the l2 term +PROBLEM = 1; +prox2 = { prox, proj_l2(lambda_1) }; +linearF2 = diag( [ones(d,1);0] ); +opts.errFcn = @(f,d,x) norm( x - [SLOPE{PROBLEM};INTERCEPT{PROBLEM}] ); +opts.tol = 1e-15; +opts.continuation = false; +[ak,out,optsOut] = tfocs_SCD([],{linearF,[];linearF2,[]}, prox2, mu,[],[],opts); + + + + +%% +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/largescale/test_sBPDN_W_largescale.m b/examples/largescale/test_sBPDN_W_largescale.m new file mode 100644 index 0000000..5cfdb1e --- /dev/null +++ b/examples/largescale/test_sBPDN_W_largescale.m @@ -0,0 +1,149 @@ +%{ + Tests the "analysis" form of basis pursuit de-noising + + min_x ||Wx||_1 +s.t. + || A(x) - b || <= eps + +This version uses a realistic problem + +see also test_sBP.m and test_sBPDN.m and test_sBPDN_W.m + +This example requires PsiTransposeWFF.m and PsiWFF.m (should be included in this directory) + +%} + +% Before running this, please add the TFOCS base directory to your path + + +HANDEL = load('handel'); % This comes with Matlab +x_original = HANDEL.y; % It's a clip from Handel's "Hallelujah" +FS = HANDEL.Fs; % The sampling frequency in Hz + +PLAY = input('Play the original music clip? (y/n) ','s'); +if strcmpi(PLAY,'y') || strcmpi(PLAY,'yes') + sound( x_original ); +end +N = length(x_original); +% Simplest to use signals of size 2^k for any integer k > 0 +% So, padd the signal with zeros at the end +k = nextpow2(N); +k = k-1; +x_original = [x_original(1:min(N,2^k)); zeros(2^k-N,1) ]; +N_short = N; +N = 2^k; + +% We'll perform denoising. Measurement is the identity. +M = N; +Af = @(x) x; +At = Af; +normA = 1; +A = linop_handles( [M,N], Af, At ); +% linop_test(A) + +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); + +snr = 10; % in dB +b_exact = Af(x_original); +x_noisy = myAwgn(x_original,snr); +b = Af(x_noisy); + +PLAY = input('Play the noisy music clip? (y/n) ','s'); +if strcmpi(PLAY,'y') || strcmpi(PLAY,'yes') + sound( x_noisy ); +end + +EPS = norm( b - b_exact ); +fprintf('||b-Ax||/||b|| is %.2f\n', EPS/norm(b) ); + +%% SETUP AN ANALYSIS OPERATOR +% The PsiWFF and PsiTransposeWFF code is a Gabor frame +% (i.e. a short-time Fourier transform) +% written by Peter Stobbe +% +% PsiWFF is the synthesis operator, and acts on coefficients +% PsiTransposeWFF is the analysis operator, and acts on signals +gMax = 0; +% gMax = -8; % 2.6e-1 er with -8 +gLevels = 1 - gMax; +tRedundancy = 1; +fRedundancy = 0; + +gWindow = 'isine'; +logN = log2(N); +psi = @(y) PsiWFF(y,gWindow,logN,logN-gLevels,logN+gMax,tRedundancy,fRedundancy); +psiT = @(x) PsiTransposeWFF(x,gWindow,logN,logN-gLevels,logN+gMax,tRedundancy,fRedundancy); + +x = x_original; +y = psiT(x); +x2 = psi(y); +d = length(y); +% The frame is tight, so the psuedo-inverse is just the transpose: +fprintf('Error in PSI(PSI^* x ) - x is %e; N = %d, d = %d, d/N = %.1f\n', ... + norm(x-x2),N,d,d/N); + +normW = 1; +W = linop_handles([d,N], psiT, psi ); +linop_test(W) +%% Call the TFOCS solver + +x0 = x_noisy; +mu = .5*norm(psiT(x0),Inf); +norm_x_orig = norm(x_original); +er_signal = @(x) norm(x-x_original)/norm_x_orig; + +opts = []; +opts.errFcn = @(f,dual,primal) er_signal(primal); +opts.maxIts = 80; +opts.tol = 1e-6; +opts.normA2 = normA^2; +opts.normW2 = normW^2; % both of these are 1 +opts.printEvery = 20; +z0 = []; % we don't have a good guess for the dual +tic; + +W_weighted = W; +% do some reweighting +for rw = 1:2 + + % This is the old method of doing continuation. The current version + % supports a simpler way to make the solver use continuation; + % for an example, see "example_LinearProgram.m" in the examples/smallscale directory. + solver = @(mu,x0,z0,opts) solver_sBPDN_W( A, W_weighted, b, EPS, mu, x0, z0, opts ); + contOpts = []; + contOpts.maxIts = 1; + [ x, out, optsOut ] = continuation( solver, mu, x0, z0, opts,contOpts ); + + % update weights + coeff = psiT(x); + cSort = sort(abs(coeff),'descend'); + cutoff = cSort( find( cumsum(cSort.^2)/sum(cSort.^2) >= .9, 1 ) ); + weights = 1./( abs(coeff) + cutoff ); + weightsMatrix = spdiags(weights,0,d,d); + W_weighted = linop_compose( weightsMatrix , W ); + opts.normW2 = []; % let the solver estimate it for us + + x0 = x; + z0 = cell( out.dual ); +end + + +time_TFOCS = toc; + +fprintf('Denoised signal has error %.2e, noisy signal has error %.2e\n',... + er_signal(x), er_signal(x_noisy) ); +%% +PLAY = input('Play the recovered signal music clip? (y/n) ','s'); +if strcmpi(PLAY,'y') || strcmpi(PLAY,'yes') + sound( x ); +end +PLAY = input('Play the noisy music clip? (y/n) ','s'); +if strcmpi(PLAY,'y') || strcmpi(PLAY,'yes') + sound( x_noisy ); +end + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/largescale/test_sTV_Analysis_largescale.m b/examples/largescale/test_sTV_Analysis_largescale.m new file mode 100644 index 0000000..995365b --- /dev/null +++ b/examples/largescale/test_sTV_Analysis_largescale.m @@ -0,0 +1,268 @@ +%{ + Tests total-variation problem and l1 analysis + on a large-scale example + + min_x alpha*||x||_TV + beta*||Wx||_1 +s.t. + || A(x) - b || <= eps + +where W is a wavelet operator. +The solvers solve a regularized version + +see also test_sBP.m and test_sBPDN_W.m and test_sTV.m and test_sTV_largescale.m + +This version has no reference solution. + +This code uses continuation + +Requires wavelet toolbox and image processing toolbox + +%} + +% Before running this, please add the TFOCS base directory to your path + + +% Generate a new problem + +randn('state',245); +rand('state',245); + +% image = 'phantom'; +image = 'bugsbunny'; + +switch image + case 'phantom' + n = 256; + n1 = n; + n2 = n; + % n2 = n-1; % testing the code with non-square signals + N = n1*n2; + n = max(n1,n2); + x = phantom(n); + x = x(1:n1,1:n2); + +% snr = -5; % SNR in dB + snr = 5; + case 'bugsbunny' + % Image is from wikimedia commons: + % http://commons.wikimedia.org/wiki/File:Falling_hare_bugs.jpg + load BugsBunny + x = bugs; + [n1,n2] = size(x); + N = n1*n2; + + snr = 15; % SNR in dB +end +x_original = x; + +mat = @(x) reshape(x,n1,n2); +vec = @(x) x(:); + +% -- Choose what kind of measurements you want: +measurements = 'identity'; % for denoising + +switch measurements + case 'identity' + M = N; + Af = @(x) x; + At = @(x) x; +end + +% treat X as n1*n2 x 1 vector? +VECTOR = true; +% or treat X as n1 x n2 matrix? +% VECTOR = false; + +if VECTOR + A = linop_handles([M,N], Af, At ); + VEC = vec; +else + Af = @(x) vec(Af(x)); + At = @(x) mat(At(x)); + A = linop_handles({[n1,n2],[M,1]}, Af, At ); + VEC = @(x) x; +end + + +b_original = Af(vec(x_original)); + +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); + +% Either add white noise: +x_noisy = myAwgn( x_original, snr ); +% Or add a random mask: +% x_temp = myAwgn( x_original, 0); +% index = randperm(N); +% index = index( 1 : round(.2*N) ); +% x_noisy = x_original; +% x_noisy(index) = x_temp(index); + +b = Af( vec(x_noisy) ); +EPS = .9*norm(b-b_original); +figure(2); imshow( [x_original, x_noisy] ); + +%% Setup a sparsifying dictionary to use for denoising: +% Choose an extension mode; see "dwtmode". This makes things square +dwtmode('per'); +X = x_original; +Xbp = x_noisy; +maxI = 1; % max pixel value +PSNR = @(x) 20*log10(maxI*sqrt(N)/norm(vec(x)-vec(x_original) ) ); + +% symmetric 9/7 biothogonal wavelets are what JPEG-2000 uses +waveletType = 'bior4.4'; % 9/7 wavelet +nLevels = min( 1, wmaxlev(size(Xbp),waveletType) ); +[c,s] = wavedec2(Xbp,nLevels,waveletType); % see 'wfilters' for options. Needs wavelet toolbox +PLOT = true; +if PLOT + D = detcoef2( 'h', c, s, 1 ); + D = [D,detcoef2( 'd', c, s, 1 )]; + a = appcoef2( c, s, 'db3', 1); + D = [D;detcoef2( 'v', c, s, 1 ), a]; + cSort = sort(abs(c),'descend'); + + % Try wavelet hard thresholding to denoise + erBest = -Inf; + for gamma = .81:.01:.999 + ind = [find( cumsum(cSort.^2)/sum(cSort.^2) > gamma ),N]; + cutoff = cSort(ind(1)); + c2 = c; + c2 = c.*( abs(c)>cutoff); + X_hat = waverec2(c2,s,waveletType); + er = PSNR(X_hat); +% fprintf('Gamma is %.3f, PSNR is %.2f\n',gamma,er ); + if er > erBest, erBest = er; gammaBest = gamma; end + end +% gammaBest = .9; + ind = [find( cumsum(cSort.^2)/sum(cSort.^2) > gammaBest ),N]; + cutoff = cSort(ind(1)); + c2 = c.*( abs(c)>cutoff); + X_hat = waverec2(c2,s,waveletType); + + figure(2); + imshow( [X,Xbp,X_hat] ); + title(sprintf('Original\t\tNoisy (PSNR %.1f dB)\tOracle Wavelet Thresholding (PNSR %.1f dB)',... + PSNR(Xbp), PSNR(X_hat) ) ); +end + +% Forward wavelet operator +Wf = @(X) wavedec2(mat(X),nLevels,'bior4.4')'; +W_invF = @(c) vec(waverec2(vec(c),s,'bior4.4') ); +pinvW = W_invF; + +Wt = @(c) vec(waverec2(vec(c),s,'rbio4.4') ); +pinvWt = @(X) wavedec2(mat(X),nLevels,'rbio4.4')'; +d = length(c); +if d ~= N, disp('warning: wavelet transform not square'); end + +W_wavelet = linop_handles([N,N], Wf, Wt ); +normWavelet = linop_test(W_wavelet); +%% +mu = 1e-3*norm( linop_TV(x_original) ,Inf); +x_ref = x_original; + +% imshow( [x_original, x_noisy] ); + +sz = A([],0); +M = sz{2}(1); +N = sz{1}(1); +[n1,n2] = size(x_original); +norm_x_ref = norm(x_ref,'fro'); +er_ref = @(x) norm(vec(x)-vec(x_ref))/norm_x_ref; + +fprintf('\tA is %s, %d x %d, signal has SNR %d dB\n', measurements, M, N, round(snr) ); + +%% Call the TFOCS solver +er = er_ref; +opts = []; +opts.restart = 1000; +opts.errFcn = @(f,dual,primal) er(primal); +opts.maxIts = 100; +opts.printEvery = 20; +opts.tol = 1e-4; +if strcmpi(measurements,'identity') + x0 = x_noisy; +else + x0 = At(b); +end +z0 = []; % we don't have a good guess for the dual +if VECTOR + W_tv = linop_TV( [n1,n2] ); +else + W_tv = linop_TV( {n1,n2} ); +% z0 = {zeros(M,1), zeros(n1*n2,1) }; +end +normTV = linop_TV( [n1,n2], [], 'norm' ); +opts.normW12 = normTV^2; +opts.normW22 = normWavelet^2; +opts.normA2 = 1; % "A" is the identity + + +% Make sure we didn't do anything bad +% linop_test( A ); +% linop_test( W ); + +contOpts = []; +contOpts.maxIts = 4; +contOpts.betaTol = 2; + + +% -- First, solve just via wavelet -- +disp('============ WAVELETS ONLY ==============='); +opts.normW2 = normWavelet^2; +solver = @(mu,x0,z0,opts) solver_sBPDN_W( A, W_wavelet, b, EPS, mu, x0, z0, opts); +[ x, out, optsOut ] = continuation(solver,5*mu,VEC(x0),z0,opts, contOpts); +X_wavelets = mat(x); + +% -- Second, solve just via TV -- +disp('============ TV ONLY ====================='); +opts.normW2 = normTV^2; +solver = @(mu,x0,z0,opts) solver_sBPDN_W( A, W_tv, b, EPS, mu, x0, z0, opts); +[ x, out, optsOut ] = continuation(solver,5*mu,VEC(x0),z0,opts, contOpts); +X_tv = mat(x); + +% -- Third, combine wavelets and tv -- +% -- Here is what we are solving -- +% minimize alpha*||x||_TV + beta*||W_wavelet(x)||_1 +% subject to ||x-x_noisy|| <= EPS +alpha = 1; +beta = .1; + +disp('============ WAVELETS AND TV ============='); +solver = @(mu,x0,z0,opts) solver_sBPDN_WW( A, alpha, W_tv, beta, W_wavelet, b, EPS, mu,x0, z0, opts); +[ x, out, optsOut ] = continuation(solver,5*mu,VEC(x0),z0, opts,contOpts); +X_wavelets_tv = mat(x); + +%% +figure(1); clf; +splot = @(n) subplot(2,3,n); + +splot(1); +imshow(x_original); +title(sprintf('Noiseless image, PSNR %.1f dB', Inf )); + +splot(2); +imshow(x_noisy); +title(sprintf('Noisy image, PSNR %.1f dB', PSNR(x_noisy) )); + +splot(3); +imshow(X_hat); +title(sprintf('Oracle wavelet thresholding, PSNR %.1f dB', PSNR(X_hat) )); + +splot(4); +imshow(X_wavelets); +title(sprintf('Wavelet regularization, PSNR %.1f dB', PSNR(X_wavelets) )); + +splot(5); +imshow(X_tv); +title(sprintf('TV regularization, PSNR %.1f dB', PSNR(X_tv) )); + +splot(6); +imshow(X_wavelets_tv); +title(sprintf('Wavelet + TV regularization, PSNR %.1f dB', PSNR(X_wavelets_tv) )); + +%% +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/largescale/test_sTV_largescale.m b/examples/largescale/test_sTV_largescale.m new file mode 100644 index 0000000..4e42ecd --- /dev/null +++ b/examples/largescale/test_sTV_largescale.m @@ -0,0 +1,154 @@ +%{ + Tests total-variation problem on a large-scale example + + min_x ||x||_TV +s.t. + || A(x) - b || <= eps + +The solvers solve a regularized version + +see also test_sBP.m and test_sBPDN_W.m and test_sTV.m + +This version has no reference solution. +Requires image processing toolbox + +%} + +% Before running this, please add the TFOCS base directory to your path + + +% Generate a new problem + +randn('state',245); +rand('state',245); + +n = 256; +n1 = n; +n2 = n; +% n2 = n-1; % testing the code with non-square signals +N = n1*n2; +n = max(n1,n2); +x = phantom(n); +x = x(1:n1,1:n2); +x_original = x; + +mat = @(x) reshape(x,n1,n2); +vec = @(x) x(:); + +% -- Choose what kind of measurements you want: +measurements = 'identity'; % for denoising +% measurements = 'partialFourier'; % partial 2D DCT + +switch measurements + case 'identity' + M = N; + Af = @(x) x; + At = @(x) x; + case 'partialFourier' + + M = round( N / 4 ); + % omega = randsample(N,M); + omega = randn(N,1); + [temp,omega] = sort(omega); + omega = sort( omega(1:M) ); + downsample = @(x) x(omega); + SS.type = '()'; SS.subs{1} = omega; SS.subs{2} = ':'; + upsample = @(x) subsasgn( zeros(N,size(x,2)),SS,x); + + + % take partial 2D DCT measurements. We have Af(At) = identity (not + % vice-versa of course). Randomly permuted. + rp = randperm(N); + [~,rp_inv] = sort(rp); + rpF = @(x) x(rp); + rp_invF = @(x) x(rp_inv); + + my_dct2 = @(x) dct(dct(x).').'; + my_idct2= @(x) idct(idct(x).').'; + Af = @(x) downsample(vec( my_dct2( mat(rpF(mat(x) )) ) ) ); + At = @(y) vec( rp_invF( my_idct2( mat( upsample( y ) ) ) ) ); +end + +% treat X as n1*n2 x 1 vector? +VECTOR = true; +% or treat X as n1 x n2 matrix? +% VECTOR = false; + +if VECTOR + A = linop_handles([M,N], Af, At ); + VEC = vec; +else + Af = @(x) vec(Af(x)); + At = @(x) mat(At(x)); + A = linop_handles({[n1,n2],[M,1]}, Af, At ); + VEC = @(x) x; +end + +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); + +b_original = Af(vec(x_original)); +snr = 10; % SNR in dB +x_noisy = myAwgn( x_original, snr ); +b = Af( vec(x_noisy) ); +EPS = norm(b-b_original); + +mu = 1e-5*norm( linop_TV(x_original) ,Inf); +x_ref = x_original; + +imshow( [x_original, x_noisy] ); + +sz = A([],0); +M = sz{2}(1); +N = sz{1}(1); +[n1,n2] = size(x_original); +norm_x_ref = norm(x_ref,'fro'); +er_ref = @(x) norm(vec(x)-vec(x_ref))/norm_x_ref; + +fprintf('\tA is %s, %d x %d, signal has SNR %d dB\n', measurements, M, N, round(snr) ); + +%% Call the TFOCS solver +er = er_ref; +opts = []; +opts.restart = 1000; +opts.errFcn = @(f,dual,primal) er(primal); +opts.maxIts = 250; +if strcmpi(measurements,'identity') + x0 = x_ref; +else + x0 = At(b); +end +z0 = []; % we don't have a good guess for the dual +if VECTOR + W = linop_TV( [n1,n2] ); +else + W = linop_TV( {n1,n2} ); +end +normW = linop_TV( [n1,n2], [], 'norm' ); +opts.normW2 = normW^2; + +% Make sure we didn't do anything bad +% linop_test( A ); +% linop_test( W ); + + +tic; +% [ x, out, optsOut ] = solver_sBPDN_W( A, W, b, EPS, mu, VEC(x0), z0, opts +% ); +solver = @(mu,x0,z0,opts) solver_sBPDN_W( A, W, b, EPS, mu, x0, z0, opts); +[ x, out, optsOut ] = continuation(solver,5*mu,x0(:),z0,opts); + +time_TFOCS = toc; +fprintf('Solution has %d nonzeros. Error vs. original solution is %.2e\n',... + nnz(x), er(x) ); + +x = mat(x); +imshow( [x_original, x_noisy, x] ); +maxI = 1; % max pixel value +PSNR = @(x) 20*log10(maxI*sqrt(N)/norm(vec(x)-vec(x_original) ) ); +title(sprintf('No denoising, PSNR is %.1f dB; TV denoising, PSNR is %.1f dB',... + PSNR( x_noisy ), PSNR( x )) ); +%% +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/README.txt b/examples/smallscale/README.txt new file mode 100644 index 0000000..4525db0 --- /dev/null +++ b/examples/smallscale/README.txt @@ -0,0 +1,7 @@ +This directory contains basic tests that have little *scientific* +value, but demonstrate how to use different solvers. + +Most of the tests use a stored reference solution (generated +from an interior-point solver) to measure error. + +These are generally small-scale tests, designed as a proof-of-concept. diff --git a/examples/smallscale/project_WeightedNorm.m b/examples/smallscale/project_WeightedNorm.m new file mode 100644 index 0000000..2a2c589 --- /dev/null +++ b/examples/smallscale/project_WeightedNorm.m @@ -0,0 +1,62 @@ +%{ + +It is very quick and basically closed-form to project onto ||x||_1 + +e.g. Proj_x0 = argmin_{ ||x||_1 <= 1 } 1/2|| x - x0 ||^2 + +but it is hard to project onto the set ||Wx||_1 <= 1, for general W +(i.e. non-invertible) + +Here we show how to solve the weighted projection with an iterative method + +%} + +N = 100; +x0 = randn(N,1); +W = randn(2*N,N); +%% project x0 onto l1 ball, no weighting (easy: one-step) +tau = .8*norm(x0,1); % make sure x0 isn't already feasibly +projection = proj_l1(tau); +[value,projX] = projection(x0,1); +fprintf('tau - ||x||_1 is %.2e\n', tau - norm(projX,1) ); + + +%% project x0 onto l1 ball, with weighting (harder: iterate) +tau = .92*norm(W*x0,1); + +mu = 1; % any value works +e = abs(eig(W'*W)/mu); +L = max(e); +strngCvxty = min(e); +opts = []; +opts.alg = 'AT'; +% opts.alg = 'N83'; +% opts.alg = 'GRA'; +opts.tol = 1e-12; +opts.maxits = 500; + +opts.L0 = L; +opts.Lexact = L; +opts.mu = strngCvxty; + +% opts.beta = 1; % prevents backtracking +opts.printEvery = 5; + +% opts.restart = 5; + +p = prox_linf(tau); % project onto the ||z||_1 <= tau ball +[projWX,outData,optsOut] = tfocs_SCD( [],W,p, mu, x0, [], opts ); + +fprintf('tau - ||Wx||_1 is %.2e, and x0 and projected version differ by %.2e\n', ... + tau - norm(W*projWX,1), norm(x0-projWX)/norm(x0) ); + +% Check that we are within allowable bounds +if abs(tau - norm(W*projWX,1)) < 1e-10 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. \ No newline at end of file diff --git a/examples/smallscale/reference_solutions/LMI.mat b/examples/smallscale/reference_solutions/LMI.mat new file mode 100644 index 0000000..1a80d09 Binary files /dev/null and b/examples/smallscale/reference_solutions/LMI.mat differ diff --git a/examples/smallscale/reference_solutions/LMI_complex.mat b/examples/smallscale/reference_solutions/LMI_complex.mat new file mode 100644 index 0000000..4a54d21 Binary files /dev/null and b/examples/smallscale/reference_solutions/LMI_complex.mat differ diff --git a/examples/smallscale/reference_solutions/LP.mat b/examples/smallscale/reference_solutions/LP.mat new file mode 100644 index 0000000..a1cbb9e Binary files /dev/null and b/examples/smallscale/reference_solutions/LP.mat differ diff --git a/examples/smallscale/reference_solutions/LP_box.mat b/examples/smallscale/reference_solutions/LP_box.mat new file mode 100644 index 0000000..d34b583 Binary files /dev/null and b/examples/smallscale/reference_solutions/LP_box.mat differ diff --git a/examples/smallscale/reference_solutions/SDP.mat b/examples/smallscale/reference_solutions/SDP.mat new file mode 100644 index 0000000..f742b2b Binary files /dev/null and b/examples/smallscale/reference_solutions/SDP.mat differ diff --git a/examples/smallscale/reference_solutions/SDP_complex.mat b/examples/smallscale/reference_solutions/SDP_complex.mat new file mode 100644 index 0000000..f061a80 Binary files /dev/null and b/examples/smallscale/reference_solutions/SDP_complex.mat differ diff --git a/examples/smallscale/reference_solutions/basispursuit_WW_problem1_smoothed_noisy.mat b/examples/smallscale/reference_solutions/basispursuit_WW_problem1_smoothed_noisy.mat new file mode 100644 index 0000000..778a0b5 Binary files /dev/null and b/examples/smallscale/reference_solutions/basispursuit_WW_problem1_smoothed_noisy.mat differ diff --git a/examples/smallscale/reference_solutions/basispursuit_W_problem1_smoothed_noisy.mat b/examples/smallscale/reference_solutions/basispursuit_W_problem1_smoothed_noisy.mat new file mode 100644 index 0000000..a130a18 Binary files /dev/null and b/examples/smallscale/reference_solutions/basispursuit_W_problem1_smoothed_noisy.mat differ diff --git a/examples/smallscale/reference_solutions/basispursuit_problem1_noisy.mat b/examples/smallscale/reference_solutions/basispursuit_problem1_noisy.mat new file mode 100644 index 0000000..6feadd2 Binary files /dev/null and b/examples/smallscale/reference_solutions/basispursuit_problem1_noisy.mat differ diff --git a/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noiseless.mat b/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noiseless.mat new file mode 100644 index 0000000..f8f4aa5 Binary files /dev/null and b/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noiseless.mat differ diff --git a/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noiseless_nonnegative.mat b/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noiseless_nonnegative.mat new file mode 100644 index 0000000..4e96d7b Binary files /dev/null and b/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noiseless_nonnegative.mat differ diff --git a/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noisy.mat b/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noisy.mat new file mode 100644 index 0000000..ae4120c Binary files /dev/null and b/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noisy.mat differ diff --git a/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noisy_complex.mat b/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noisy_complex.mat new file mode 100644 index 0000000..a41cb1d Binary files /dev/null and b/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noisy_complex.mat differ diff --git a/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noisy_complex_2.mat b/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noisy_complex_2.mat new file mode 100644 index 0000000..a48f6df Binary files /dev/null and b/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noisy_complex_2.mat differ diff --git a/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noisy_nonnegative.mat b/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noisy_nonnegative.mat new file mode 100644 index 0000000..f2f697f Binary files /dev/null and b/examples/smallscale/reference_solutions/basispursuit_problem1_smoothed_noisy_nonnegative.mat differ diff --git a/examples/smallscale/reference_solutions/blocknorm_smoothed_noisy.mat b/examples/smallscale/reference_solutions/blocknorm_smoothed_noisy.mat new file mode 100644 index 0000000..804fb8d Binary files /dev/null and b/examples/smallscale/reference_solutions/blocknorm_smoothed_noisy.mat differ diff --git a/examples/smallscale/reference_solutions/complicatedProblem1.mat b/examples/smallscale/reference_solutions/complicatedProblem1.mat new file mode 100644 index 0000000..c0caa1c Binary files /dev/null and b/examples/smallscale/reference_solutions/complicatedProblem1.mat differ diff --git a/examples/smallscale/reference_solutions/dantzig_problem1_smoothed_noisy.mat b/examples/smallscale/reference_solutions/dantzig_problem1_smoothed_noisy.mat new file mode 100644 index 0000000..a0944e4 Binary files /dev/null and b/examples/smallscale/reference_solutions/dantzig_problem1_smoothed_noisy.mat differ diff --git a/examples/smallscale/reference_solutions/lasso_problem1_noisy.mat b/examples/smallscale/reference_solutions/lasso_problem1_noisy.mat new file mode 100644 index 0000000..0837a18 Binary files /dev/null and b/examples/smallscale/reference_solutions/lasso_problem1_noisy.mat differ diff --git a/examples/smallscale/reference_solutions/nuclearNorm_problem1_noiseless.mat b/examples/smallscale/reference_solutions/nuclearNorm_problem1_noiseless.mat new file mode 100644 index 0000000..46ad420 Binary files /dev/null and b/examples/smallscale/reference_solutions/nuclearNorm_problem1_noiseless.mat differ diff --git a/examples/smallscale/reference_solutions/ordered_asso_problem1_noisy.mat b/examples/smallscale/reference_solutions/ordered_asso_problem1_noisy.mat new file mode 100644 index 0000000..fcc5848 Binary files /dev/null and b/examples/smallscale/reference_solutions/ordered_asso_problem1_noisy.mat differ diff --git a/examples/smallscale/reference_solutions/traceLS_problem1_noisy.mat b/examples/smallscale/reference_solutions/traceLS_problem1_noisy.mat new file mode 100644 index 0000000..d84dc51 Binary files /dev/null and b/examples/smallscale/reference_solutions/traceLS_problem1_noisy.mat differ diff --git a/examples/smallscale/reference_solutions/traceLS_problem2_noisy.mat b/examples/smallscale/reference_solutions/traceLS_problem2_noisy.mat new file mode 100644 index 0000000..85cd100 Binary files /dev/null and b/examples/smallscale/reference_solutions/traceLS_problem2_noisy.mat differ diff --git a/examples/smallscale/reference_solutions/tv_problem1_smoothed_noisy.mat b/examples/smallscale/reference_solutions/tv_problem1_smoothed_noisy.mat new file mode 100644 index 0000000..9b885fd Binary files /dev/null and b/examples/smallscale/reference_solutions/tv_problem1_smoothed_noisy.mat differ diff --git a/examples/smallscale/test_LASSO.m b/examples/smallscale/test_LASSO.m new file mode 100644 index 0000000..dc512de --- /dev/null +++ b/examples/smallscale/test_LASSO.m @@ -0,0 +1,95 @@ +%{ + Tests the LASSO ( aka L1 regularized Least Squares) problem + + min_x lambda*||x||_1 + .5||A(x)-b||_2^2 + +see also test_sBPDN.m + +%} + +% Before running this, please add the TFOCS base directory to your path +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); +%% +% Try to load the problem from disk +fileName = fullfile(tfocs_where,... + 'examples','smallscale','reference_solutions','lasso_problem1_noisy'); +randn('state',34324); +rand('state',34324); +N = 1024; +M = round(N/2); +K = round(M/5); +A = randn(M,N); +% give A unit-normed columns +A = bsxfun(@times,A,1./sqrt(sum(A.^2,1))); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + + % Generate a new problem + x = zeros(N,1); + T = randsample(N,K); + x(T) = randn(K,1); + + b_original = A*x; + snr = 30; % SNR in dB + sigma = 10^( (10*log10( norm(b_original)^2/N) - snr)/20 ); + b = myAwgn(b_original,snr); + x_original = x; + + lambda = 2*sigma*sqrt(2*log(N)); % for when A has unit-normed columns + x0 = zeros(N,1); + + % get reference via CVX + tic + cvx_begin + cvx_precision best + variable xcvx(N,1) + minimize lambda*norm(xcvx,1) + sum_square( A*xcvx - b)/2 + cvx_end + time_IPM = toc; + x_ref = xcvx; + obj_ref = lambda*norm(x_ref,1) + sum_square( A*xcvx - b)/2; + + save(fileName,'x_ref','b','x_original','lambda',... + 'sigma','b_original','obj_ref','x0','time_IPM','snr'); + fprintf('Saved data to file %s\n', fileName); + +end + +x_ref = x_ref.*( abs(x_ref) > 1e-8 ); +[M,N] = size(A); +K = nnz(x_original); +norm_x_ref = norm(x_ref); +norm_x_orig = norm(x_original); +er_ref = @(x) norm(x-x_ref)/norm_x_ref; +er_signal = @(x) norm(x-x_original)/norm_x_orig; +resid = @(x) norm(A*x-b)/norm(b); % change if b is noisy + +fprintf('\tA is %d x %d, original signal has %d nonzeros\n', M, N, K ); +fprintf('\tl1-norm solution and original signal differ by %.2e (lambda = %.2e)\n', ... + norm(x_ref - x_original)/norm(x_original),lambda ); + +%% Call the TFOCS solver +er = er_ref; % error with reference solution (from IPM) +opts = struct('restart',-Inf,'tol',1e-13,'maxits',1000); +opts.errFcn = { @(f,primal) er(primal), ... + @(f,primal) f - obj_ref }; +tic; +[ x, out, optsOut ] = solver_L1RLS( A, b, lambda, x0, opts ); +time_TFOCS = toc; + +fprintf('Solution has %d nonzeros. Error vs. IPM solution is %.2e\n',... + nnz(x), er(x) ); + +% Check that we are within allowable bounds +if out.err(end,1) < 1e-7 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_LMI.m b/examples/smallscale/test_LMI.m new file mode 100644 index 0000000..8a098c8 --- /dev/null +++ b/examples/smallscale/test_LMI.m @@ -0,0 +1,134 @@ +%{ + Via smoothing, TFOCS can solve Linear Matrix Inequalities (LMI) + (these are equivalent to the dual of a SDP in standard form) + + min_y + s.t. A_0 + sum_i=1^M A_i * y(i) >= 0 + +where ">=" refers to the positive semi-definite cone. +"b" and "y" are vectors of length M, +and A_i (i = 0, 1, ..., M) +are symmetric or Hermian matrices of size N x N +(where M <= N^2 ) + +TFOCS assumes that all the A_i matrices (i=1,...,M; the A_0 matrix is +handled separately) are efficiently stored in one big "A" matrix +so that we can compactly replace the sum_i A_i * y(i) +with mat( A'*y ), where mat() is a reshaping operator. +The code below shows how we do this. + + See also test_SDP.m + + If we identify A0 with -C, then this is the dual of the SDP in standard form. + +%} + +% Add TFOCS to your path. + +randn('state',9243); rand('state',23432); + +N = 30; +M = round(N^2/4); + +symmetrize = @(X) (X+X')/2; +vec = @(X) X(:); +mat = @(y) reshape(y,N,N); + +Acell = cell(M,1); +for m = 1:M + Acell{m} = symmetrize( randn(N) ); +end + +% Put "Acell" into a matrix: +A = zeros(M,N^2); +for m = 1:M + A(m,:) = vec( Acell{m} ); +end +normA = normest(A,1e-2); + +b = randn(M,1); +y = randn(M,1); +A0 = mat(A'*y); % want A0 in range of A', otherwise infeasible or unbounded + +fileName = fullfile('reference_solutions','LMI'); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + cvx_begin + variable y(M) + minimize( b'*y ) + subject to + A0 + mat(A'*y) >= 0 + cvx_end + y_cvx = y; + save(fullfile(pwd,fileName),'y_cvx'); + fprintf('Saved data to file %s\n', fileName); +end + +%% solve via TFOCS +mu = 1e-3; +opts = []; +opts.errFcn = @(f,d,y) norm( y - y_cvx)/norm(y_cvx); + +y0 = []; z0 = []; +[yy, out, opts ] = solver_sLMI( A0, A, b, mu, y0, z0, opts ); + +% Check that we are within allowable bounds +if out.err(end) < 1e-8 + disp('Everything is working'); +else + error('Failed the test'); +end + +%% Version with complex variables: +randn('state',9243); rand('state',23432); +N = 30; +M = round(N^2/4); +Acell = cell(M,1); +for m = 1:M + Acell{m} = symmetrize( randn(N) + 1i*randn(N) ); +end +A = zeros(M,N^2); +for m = 1:M + A(m,:) = vec( Acell{m} ); +end +normA = normest(A,1e-2); + +b = randn(M,1); % this should always be real +y = randn(M,1); % this should always be real +A0 = mat(A'*y); % this may be complex + +fileName = 'reference_solutions/LMI_complex'; +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + cvx_begin + variable y(M) + minimize( b'*y ) + subject to + A0 + mat(A'*y) == semidefinite(N) + cvx_end + y_cvx = y; + save(fullfile(pwd,fileName),'y_cvx'); + fprintf('Saved data to file %s\n', fileName); +end + +% and solve via TFOCS: +mu = 1e-3; +opts = []; +opts.errFcn = @(f,d,y) norm( y - y_cvx)/norm(y_cvx); +y0 = []; z0 = []; +[yy, out, opts ] = solver_sLMI( A0, A, b, mu, y0, z0, opts ); + +% Check that we are within allowable bounds +if out.err(end) < 1e-8 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_LinearProgram.m b/examples/smallscale/test_LinearProgram.m new file mode 100644 index 0000000..a50e22f --- /dev/null +++ b/examples/smallscale/test_LinearProgram.m @@ -0,0 +1,153 @@ +%{ + Via smoothing, TFOCS can solve a Linear Program (LP) in standard form: + +(P) min c'x s.t. x >= 0, Ax=b + + + This extends to SDP as well (see test_SDP.m ) +%} + +% Add TFOCS to your path. + +randn('state',9243); rand('state',234324'); + +N = 5000; +M = round(N/2); +x = randn(N,1)+10; +A = sprand(M,N,.01); +c = randn(N,1); +b = A*x; +if issparse(A) + normA = normest(A); +else + normA = norm(A); +end + +fileName = fullfile('reference_solutions','LP'); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + cvx_begin + variable x(N) + minimize( c'*x ) + subject to + x >= 0 + A*x == b + cvx_end + x_cvx = x; + save(fullfile(pwd,fileName),'x_cvx'); + fprintf('Saved data to file %s\n', fileName); +end + +%% solve in TFOCS + +opts = []; +opts.errFcn = {@(f,d,x) norm( x - x_cvx )/norm(x_cvx )}; % optional +% opts.errFcn{end+1} = @(f,d,x) norm( A*x - b )/norm(b ); % optional +opts.restart = 2000; +opts.continuation = true; +x0 = []; +z0 = []; +mu = 1e-2; + +opts.stopCrit = 4; +opts.tol = 1e-4; +contOpts = []; % options for "continuation" settings +% (to see possible options, type "continuation()" ) +% By changing the options, you can get much better results sometimes, +% but here we use default choices for simplicity. +% contOpts.accel = false; +% contOpts.betaTol = 10; +% contOpts.finalTol = 1e-10; +contOpts.maxIts = 10; +contOpts.initialTol = 1e-3; + +opts.normA = normA; % this will help with scaling +[x,out,optsOut] = solver_sLP(c,A,b, mu, x0, z0, opts, contOpts); +% Check that we are within allowable bounds +if out.err(end) < 7e-2 + disp('Everything is working'); +else + error('Failed the test'); +end +%% plot +figure(2); +semilogy( out.err(:,1) ); + + +%% == TFOCS can also handle LPs with box constraints ========== +randn('state',9243); rand('state',234324'); + +N = 3000; +M = round(N/2); +x = randn(N,1)+10; +A = sprand(M,N,.01); +c = randn(N,1); +b = A*x; +% There is no longer an automatic x >= 0 bound +% (if you want to impose this, incorporate it into the lower bound) +l = -2*ones(N,1); % lower bound +u = 13*ones(N,1); % upper bound +if issparse(A) + normA = normest(A); +else + normA = norm(A); +end + +fileName = fullfile('reference_solutions','LP_box'); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + cvx_begin + variable x(N) + cvx_precision best + minimize( c'*x ) + subject to + x >= l + x <= u + A*x == b + cvx_end + x_cvx = x; + save(fullfile(pwd,fileName),'x_cvx'); + fprintf('Saved data to file %s\n', fileName); +end + +%% solve box-constrained version in TFOCS + +opts = []; +opts.errFcn = {@(f,d,x) norm( x - x_cvx )/norm(x_cvx )}; % optional +opts.restart = 2000; +opts.continuation = true; +x0 = []; +z0 = []; +mu = 1e-2; + +opts.stopCrit = 4; +opts.tol = 1e-5; +contOpts = []; % options for "continuation" settings +contOpts.maxIts = 5; +contOpts.muDecrement = .8; +% contOpts.accel = false; + +opts.normA = normA; % this will help with scaling +[x,out,optsOut] = solver_sLP_box(c,A,b,l,u,mu, x0, z0, opts, contOpts); + +% Check that we are within allowable bounds +if out.err(end) < 2e-2 + disp('Everything is working'); +else + error('Failed the test'); +end + +%% plot box-constrained version +figure(2); +semilogy( out.err(:,1) ); + +%% close figures +close all + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_OrderedLASSO.m b/examples/smallscale/test_OrderedLASSO.m new file mode 100644 index 0000000..f87eaa3 --- /dev/null +++ b/examples/smallscale/test_OrderedLASSO.m @@ -0,0 +1,103 @@ +%{ + Tests the Ordered LASSO ( aka Ordered L1 regularized Least Squares) problem + + min_x sum(lambda*sort(abs(x),'descend')) + .5||A(x)-b||_2^2 + + lambda must be a vector in decreasing order, and all strictly positive + + Like the test_LASSO.m file, we will use the notation (A,b,x) + In the paper mentioned below, this corresponds to the notation (X,y,beta) + + Reference: + "Statistical Estimation and Testing via the Ordered l1 Norm" + by M. Bogdan, E. van den Berg, W. Su, and E. J. Candès, 2013 + http://www-stat.stanford.edu/~candes/OrderedL1/ + +see also test_sBPDN.m and test_LASSO.m + +%} + +% Before running this, please add the TFOCS base directory to your path +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); +%% +% Try to load the problem from disk +fileName = fullfile(tfocs_where,... + 'examples','smallscale','reference_solutions','ordered_asso_problem1_noisy'); +randn('state',34324); +rand('state',34324); +N = 1024; +M = round(N/2); +K = round(M/5); +A = randn(M,N); +% give A unit-normed columns +A = bsxfun(@times,A,1./sqrt(sum(A.^2,1))); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + + % Generate a new problem + x = zeros(N,1); + T = randsample(N,K); + x(T) = randn(K,1); + + b_original = A*x; + snr = 30; % SNR in dB + sigma = 10^( (10*log10( norm(b_original)^2/N) - snr)/20 ); + b = myAwgn(b_original,snr); + x_original = x; + + lambda = 2*sigma*sqrt(2*log(N)); % for when A has unit-normed columns + % Now, we diverge from test_LASSO.m, and make lambda a vector +% lambda = fliplr( linspace(.5*lambda,2*lambda,N) )'; + lambda = .5*lambda + 1.5*lambda*rand(N,1); lambda = sort(lambda,'descend') + + x0 = zeros(N,1); + + % We cannot get the solution via CVX easily, so solve using our method + opts = struct('restart',-Inf,'tol',1e-13,'maxits',1000, 'printEvery',10); + [ x, out, optsOut ] = solver_OrderedLASSO( A, b, lambda, x0, opts ); + x_ref = x; + obj_ref = norm(A*x-b)^2/2 + sum(lambda(:).*sort(abs(x),'descend')); + + save(fileName,'x_ref','b','x_original','lambda',... + 'sigma','b_original','obj_ref','x0','time_IPM','snr'); + fprintf('Saved data to file %s\n', fileName); + +end + +[M,N] = size(A); +K = nnz(x_original); +norm_x_ref = norm(x_ref) + 1*( norm(x_ref)==0 ); +norm_x_orig = norm(x_original) + 1*( norm(x_original)==0 ); +er_ref = @(x) norm(x-x_ref)/norm_x_ref; +er_signal = @(x) norm(x-x_original)/norm_x_orig; +resid = @(x) norm(A*x-b)/norm(b); % change if b is noisy + +fprintf('\tA is %d x %d, original signal has %d nonzeros\n', M, N, K ); +fprintf('\tl1-norm solution and original signal differ by %.2e (mean(lambda) = %.2e)\n', ... + norm(x_ref - x_original)/norm(x_original),mean(lambda) ); + +%% Call the TFOCS solver +er = er_ref; % error with reference solution (from IPM) +opts = struct('restart',-Inf,'tol',1e-13,'maxits',1000, 'printEvery',10); +opts.errFcn = { @(f,primal) er(primal), ... + @(f,primal) f - obj_ref }; +tic; +[ x, out, optsOut ] = solver_OrderedLASSO( A, b, lambda, x0, opts ); +time_TFOCS = toc; + +fprintf('Solution has %d nonzeros. Error vs. reference solution is %.2e\n',... + nnz(x), er(x) ); + +% Check that we are within allowable bounds +if out.err(end,1) < 1e-7 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_SDP.m b/examples/smallscale/test_SDP.m new file mode 100644 index 0000000..e43ad91 --- /dev/null +++ b/examples/smallscale/test_SDP.m @@ -0,0 +1,143 @@ +%{ + Via smoothing, TFOCS can solve a Semi-definite Program (SDP) in standard form: + +(P) min s.t. X >= 0, A(X)=b + + + This extends to LP as well (see test_LinearProgram.m ) + We can also solve the dual of SDP, which is LMI. See test_LMI.m +%} + +% Add TFOCS to your path. + +randn('state',9243); rand('state',234324'); + +N = 80; +M = round(N^2/2); +X = randn(N); X = X*X'; +A = sprand(M,N^2,.005); +C = randn(N); C = C + C'; +vec = @(X) X(:); +mat = @(y) reshape(y,N,N); +% Important: we want each row of A to be the vectorized form of the +% symmetric matrix A_i. Otherwise, the dual problem makes no sense. +% Here, we'll ensure this symmetry. +% (BTW, due to how Matlab stores sparse arrays, we only want to make +% column operations. So we'll operate on A' and then transpose back) +At = A'; +for row = 1:M + At(:,row) = vec( mat(At(:,row)) + mat(At(:,row))' )/2; +end +A = At'; clear At; + +b = A*vec(X); + +normA = normest(A); + +fileName = fullfile('reference_solutions','SDP'); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + cvx_begin sdp + variable X(N,N) symmetric + minimize trace( C*X ) + subject to + X >= 0 + A*vec(X) == b + cvx_end + X_cvx = X; + save(fullfile(pwd,fileName),'X_cvx'); + fprintf('Saved data to file %s\n', fileName); +end + +%% solve in TFOCS + +opts = []; +opts.errFcn = {@(f,d,X) norm( X - X_cvx,'fro')/norm(X_cvx,'fro' )}; % optional +% opts.errFcn{end+1} = @(f,d,X) norm( A*vec(X) - b )/norm(b); % optional +opts.restart = 2000; +opts.continuation = true; +x0 = []; +z0 = []; +mu = 1; + +opts.stopCrit = 4; +opts.tol = 1e-4; +contOpts = []; % options for "continuation" settings +% (to see possible options, type "continuation()" ) +% By changing the options, you can get much better results sometimes, +% but here we use default choices for simplicity. +% contOpts.accel = false; +% contOpts.betaTol = 10; +% contOpts.finalTol = 1e-10; + +opts.normA = normA; % this will help with scaling +[x,out,optsOut] = solver_sSDP(C,A,b, mu, x0, z0, opts, contOpts); +% Check that we are within allowable bounds +if out.err(end) < 5e-2 + disp('Everything is working'); +else + error('Failed the test'); +end +%% Here's another simple example, using complex valued variables +randn('state',9243); rand('state',234324'); + +N = 20; +M = round(N^2/1.3); +X = randn(N) + 1i*randn(N); X = X*X'; % ensure Hermitian and pos. semi-def. +A = sprandn(M,N^2,.01) + 1i*sprandn(M,N^2,.01); +C = randn(N)+1i*randn(N); C = C + C'; % ensure Hermitian +vec = @(X) X(:); +mat = @(y) reshape(y,N,N); +% Same as before... +At = A'; +for row = 1:M + At(:,row) = vec( mat(At(:,row)) + mat(At(:,row))' )/2; +end +A = At'; clear At; + +% Note: b should be real, but due to round errors it has as small imaginary part +b = real(A*vec(X)); + +normA = linop_normest(A); +fileName = 'reference_solutions/SDP_complex'; +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + cvx_begin sdp + variable X(N,N) hermitian % note the change from symmetric to hermitian + minimize real(trace( C*X )) % this should be real, but there are rounding errors + subject to + X >= 0 + real(A*vec(X)) == b + cvx_end + X_cvx = X; + save(fullfile(pwd,fileName),'X_cvx'); + fprintf('Saved data to file %s\n', fileName); +end + +% and solve with TFOCS +opts = []; +opts.errFcn = {@(f,d,X) norm( X - X_cvx,'fro')/norm(X_cvx,'fro' )}; +opts.continuation = true; +x0 = []; +z0 = []; +mu = 1e-3; +% opts.cmode = 'C2R'; % explicitly tell it that we are complex. not necessary though +opts.normA = normA; % this will help with scaling +opts.stopCrit = 4; +opts.tol = 1e-5; +[x,out,optsOut] = solver_sSDP(C,A,b, mu, x0, z0, opts, contOpts); + +% Check that we are within allowable bounds +if out.err(end) < 5e-2 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_TraceLS.m b/examples/smallscale/test_TraceLS.m new file mode 100644 index 0000000..b7e6e3a --- /dev/null +++ b/examples/smallscale/test_TraceLS.m @@ -0,0 +1,121 @@ +%{ +Test the trace-constrained least-squares problem with positive +semi-definite matrix variables + + minimize (1/2)*norm( A * X - b )^2 + lambda * trace( X ) + with the constraint that X is positive semi-definite ( X >= 0 ) +%} + +% Try to load the problem from disk +fileName = fullfile('reference_solutions','traceLS_problem1_noisy'); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + % Generate a new problem + + randn('state',sum('Trace')); + rand('state',sum('Trace2')); + + N = 30; R = 2; + + df = R*N; + oversample = 5; + Left = randn(M,R); + Right = Left; + k = round(oversample*df); + k = min( k, round(.8*N*N) ); + omega = randperm(N*N); + omega = sort(omega(1:k)).'; + + X_original = Left*Right'; % the "original" signal -- may not be optimal value + b_original = X_original(omega); + EPS = .001; % noise level + noise = EPS * randn(k,1); + b = b_original + noise; + lambda = 1e-2; + objective = @(X) sum_square( X(omega) - b )/2 + lambda*trace(X); + obj_original = objective(X_original); + + % get reference via CVX + cvx_begin + cvx_precision best + cvx_quiet true + variable Xcvx(N,N) + minimize objective(Xcvx) + subject to + Xcvx == semidefinite(N) + cvx_end + X_reference = Xcvx; % the nuclear norm minimizer + obj_reference = objective(X_reference); + fprintf('Difference between convex solution and original signal: %.2e\n', ... + norm( Xcvx - X_original, 'fro' ) ); + save(fileName,'X_original','X_reference','omega','b','obj_original',... + 'Left','EPS','b_original','R','obj_reference','lambda'); + fprintf('Saved data to file %s\n', fileName); + +end + +[M,N] = size(X_reference); +norm_X_reference = norm(X_reference,'fro'); +er_reference = @(x) norm(x-X_reference,'fro')/norm_X_reference; +objective = @(X) norm( X(omega) - b )^2/2 + lambda*trace(X); + +[omegaI,omegaJ] = ind2sub([M,N],omega); +mat = @(x) reshape(x,M,N); +vec = @(x) x(:); + +k = length(omega); +p = k/(M*N); +df = R*(M+N-R); +fprintf('%d x %d rank %d matrix, observe %d = %.1f x df = %.1f%% entries\n',... + M,N,R,k,k/df,p*100); +fprintf(' Trace norm solution and original matrix differ by %.2e\n',... + norm(X_reference-X_original,'fro')/norm_X_reference ); +%% Solve. No smoothing is necessary! +opts = struct('tol',1e-12); +opts.errFcn = @(f,x) er_reference(x); +x0 = []; +% opts.debug = true; + +% we need to tell it how to subsample. One method: +A = linop_subsample( {[N,N],[k,1]}, omega ); + +% another method: make a matrix with the correct +% sparsity pattern and let solver_TraceLS do it for us +% A = sparse( omegaI,omegaJ,ones(k,1),N,N); + +[x,out] = solver_TraceLS( A, b, lambda, x0, opts ); +epsilon = norm( x(omega) - b ); +h=figure(); +semilogy(out.err); + +% Check that we are within allowable bounds +if out.err(end) < 1e-5 + disp('Everything is working'); +else + error('Failed the test'); +end +%% We can solve this via a smoothed nuclear norm formulation +% But it will be slower! +% Also, it will not enforce the positive semi-definite constraint, +% so it is not exactly the same. + +testOptions = {}; +opts = []; +opts.errFcn = @(f,dual,x) er_reference(x); +opts.continuation = true; +opts.stopCrit = 4; +opts.tol = 1e-6; + +mu = .01; +X0 = 0; + +% [ x, out, opts ] = solver_sNuclearBP( {M,N,omega}, b, mu, X0, [], opts ); +[ x, out, opts ] = solver_sNuclearBPDN( {M,N,omega}, b, epsilon, mu, X0, [], opts ); + +%% +close(h) +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_all.m b/examples/smallscale/test_all.m new file mode 100644 index 0000000..4e0b185 --- /dev/null +++ b/examples/smallscale/test_all.m @@ -0,0 +1,30 @@ +%{ +Runs all tests in this directory. + +Recommended if you want to be sure that TFOCS is working 100% perfectly +on your system. + +The scripts will produce an error message if anything goes wrong. +So if you don't seen any error message in red text, then everything +is working. + +%} + +w = what; +fileList = w.m; +% and exclude the current file, otherwise we go into an infinite loop: +fileList = fileList( ~strcmpi( fileList, 'test_all.m' ) ); +tm_all = tic; +for k = 1:length(fileList) + + file = fileList{k}; % this has the .m extension, which we need to remove + eval( file(1:end-2) ); + +end +fprintf('\n\n\n-- Success!! --\n'); + +toc( tm_all ) + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_blockNorm.m b/examples/smallscale/test_blockNorm.m new file mode 100644 index 0000000..cc4d023 --- /dev/null +++ b/examples/smallscale/test_blockNorm.m @@ -0,0 +1,211 @@ +%{ + Tests two common block norms + + min_X ||X||_{1,p} +s.t. + || A(X) - b || <= eps + +where ||X||_{1,p} is sum_{i=1:m} norm(X(i,:),p) +and X is a m x n matrix. "p" is either 2 or Inf. +If n = 1, then for both p = 2 and p = Inf, this reduces +to the l1 norm. + +We may think of the columns of X as a new signal. The block +norm may be useful when we believe that these columns +are quite similar, e.g. when their supports overlap significantly. + +see also test_sBPDN.m + +%} +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); + +% Before running this, please add the TFOCS base directory to your path + +% Try to load the problem from disk +fileName = fullfile('reference_solutions','blocknorm_smoothed_noisy'); +randn('state',34324); +rand('state',34324); +% N = 1024; +N = 128; +M = round(N/2); +K = round(M/5); +A = randn(M,N); +d = 10; % number of columns of the signal matrix +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + + % Generate a new problem + % Our signal model is the following: + % for each column, half of the support (i.e. K/2 entries) + % is taken from the first 1:K elements, so these will likely + % have a strong overlap. + % The second half of the support is taken from the remaining + % elements, so will have less chance of overlap. + % (May not be 1/2, but that's the rough idea) + x = zeros(N,d); + K2 = round(.6*K); + for i = 1:d + T1 = randsample(K,K2); + T2 = randsample(N-K,K-K2) + K; + x(T1,i) = randn(K2,1); + x(T2,i) = randn(K-K2,1); + end + + b_original = A*x; + snr = 10; % SNR in dB + b = myAwgn(b_original,snr); + EPS = norm(b-b_original,'fro'); + x_original = x; + err = @(X) norm(X-x_original,'fro')/norm(x_original,'fro'); + + mu = .01*norm(x,Inf); + x0 = zeros(N,d); + + % get reference via CVX + + % How would we do if we did the estimation separately? + Xcvx_separate = zeros(N,d); + for i = 1:d + cvx_begin + cvx_precision best + cvx_quiet true + variable xcvx(N,1) + minimize norm(xcvx,1) + mu/2*sum_square(xcvx-x0(:,i) ) + subject to + norm(A*xcvx - b(:,i) ) <= EPS/sqrt(d) + cvx_end + Xcvx_separate(:,i) = xcvx; + end + fprintf('Estimating each column separately, the error is %.2e\n',... + err(Xcvx_separate) ); + + + % Now, use p = 2 + p = 2; + dim = 2; + cvx_begin + cvx_precision best + cvx_quiet true + variable xcvx(N,d) + minimize sum(norms(xcvx,p,dim)) + mu/2*pow_pos( norm(xcvx-x0,'fro'), 2) + subject to + norm(A*xcvx - b,'fro' ) <= EPS + cvx_end + Xcvx_p2 = xcvx; + fprintf('Estimating using the block 1-2 norm, the error is %.2e\n',... + err(Xcvx_p2) ); + + % Now, use p = Inf + p = Inf; + dim = 2; + cvx_begin + cvx_precision best + cvx_quiet true + variable xcvx(N,d) + minimize sum(norms(xcvx,p,dim)) + mu/2*pow_pos( norm(xcvx-x0,'fro'), 2) + subject to + norm(A*xcvx - b,'fro' ) <= EPS + cvx_end + Xcvx_pInf = xcvx; + fprintf('Estimating using the block 1-Inf norm, the error is %.2e\n',... + err(Xcvx_pInf) ); + + + save(fileName,'Xcvx_separate','Xcvx_p2','Xcvx_pInf','b','x_original','mu',... + 'EPS','b_original','x0','snr'); + fprintf('Saved data to file %s\n', fileName); + +end + +err = @(X) norm(X-x_original,'fro')/norm(x_original,'fro'); +fprintf('Block norm estimation: the error between cvx solution and orignal signal is...\n'); +fprintf(' Estimating each column separately, the error is %.2e\n',... + err(Xcvx_separate) ); +fprintf(' Estimating using the block 1-2 norm, the error is %.2e\n',... + err(Xcvx_p2) ); +fprintf(' Estimating using the block 1-Inf norm, the error is %.2e\n',... + err(Xcvx_pInf) ); + +%% Call the TFOCS solver for each column separately +disp('-- Testing TFOCS against the IPM (CVX) solution, each column separately --'); +X_separate = zeros(N,d); +for i = 1:d + er = @(x) norm(x-Xcvx_separate(:,i))/norm(Xcvx_separate(:,i)); + opts = []; + opts.restart = 500; + opts.errFcn = @(f,dual,primal) er(primal); + opts.maxIts = 1000; + opts.tol = 1e-8; + opts.printEvery = 500; + opts.fid = 0; + opts.restart = 100; + z0 = []; % we don't have a good guess for the dual + + [ x, out, optsOut ] = solver_sBPDN( A, b(:,i), EPS/sqrt(d), mu, x0(:,i), z0, opts ); + + fprintf('Error vs. IPM solution is %.2e\n',er(x) ); + X_separate(:,i) = x; + + % Check that we are within allowable bounds + if er(x) > 1e-6 + error('Failed the test'); + end +end +fprintf('Overall error vs. IPM solution is %.2e\n',... + norm(X_separate-Xcvx_separate,'fro')/norm(Xcvx_separate,'fro') ); + +%% Call the TFOCS solver for p = 2 +disp('-- Testing TFOCS against the IPM (CVX) solution, block norm, p = 2 --'); +er = @(x) norm(x-Xcvx_p2,'fro')/norm(Xcvx_p2,'fro'); +opts = []; +opts.restart = 200; +opts.errFcn = @(f,dual,primal) er(primal); +opts.maxIts = 1000; +opts.tol = 1e-8; +opts.printEvery = 50; +opts.fid = 1; +z0 = []; % we don't have a good guess for the dual + +AA = A; % doesn't work "out-of-the-box", because it thnkgs + % the domain of A is N x 1 matrices (it's really N x d). +AA = linop_matrix(A, 'r2r',d ); +[x,out,opts] = tfocs_SCD( prox_l1l2, { AA, -b }, prox_l2( EPS ), mu, x0, z0, opts ); + +fprintf('Error vs. IPM solution is %.2e\n',er(x) ); + +% Check that we are within allowable bounds +if er(x) < 1e-6 + disp('Everything is working'); +else + error('Failed the test'); +end +%% Call the TFOCS solver for p = Inf +disp('-- Testing TFOCS against the IPM (CVX) solution, block norm, p = Inf --'); +er = @(x) norm(x-Xcvx_pInf,'fro')/norm(Xcvx_pInf,'fro'); +opts = []; +opts.restart = 200; +opts.errFcn = @(f,dual,primal) er(primal); +opts.maxIts = 1000; +opts.tol = 1e-8; +opts.printEvery = 50; +opts.fid = 1; +z0 = []; % we don't have a good guess for the dual + +AA = A; % doesn't work "out-of-the-box", because it thnkgs + % the domain of A is N x 1 matrices (it's really N x d). +AA = linop_matrix(A, 'r2r',d ); +[x,out,opts] = tfocs_SCD( prox_l1linf(1), { AA, -b }, prox_l2( EPS ), mu, x0, z0, opts ); + +fprintf('Error vs. IPM solution is %.2e\n',er(x) ); +if er(x) < 3e-6 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_complicatedUsage.m b/examples/smallscale/test_complicatedUsage.m new file mode 100644 index 0000000..d75177a --- /dev/null +++ b/examples/smallscale/test_complicatedUsage.m @@ -0,0 +1,274 @@ +%{ +This file shows an example of using the full power of TFOCS. If you +are not familiar with basic usage of TFOCS, please see other demos +first. + +We make an example that uses: + -several variables (2), and the variables are matrices, not just vectors + -several constraints (4) + -affine operators (linear plus offset), and two of the operators + are matrix --> matrix operators + -debug mode + + +For small complicated examples like this, TFOCS is not necessarily +faster than software like CVX, since it takes more iterations than an +interior point method and there is some overhead in the TFOCS software +since the software is meant for flexibility rather than absolute speed. + +However, if you take any complicated problem and scale the size by a factor +of 100, then CVX won't be able to handle it at all. TFOCS will do +just fine (and it might even be less than 100x as slow, since now the overhead +is not significant). + + +The problem we will solve: + +min_{X1, X2} smooth1(X1)+smooth2(X2) + sum_{i=1}^4 g_i( A1_i*X1 + A2_i*X2 + B_i ) + +meaning that we have 2 variables (X1 and X2), both of which are matrices, +and each variable has its own smooth function. +For non-smooth and/or constraint terms (indexed by "i"), we have 4 functions +( i = 1,2,3,4) g_i, and each has it's own affine operator in X1 and X2. +So each of the 4 affine operators has three parts: the portion linear in X1, +the portion linear in X2, and the constant offset "B_i". + +To be explicit, the smooth functions are linear and quadratic, resp.: + +smooth1(X1) = dot( s1, X1 ) + 3.4 ("s1" is a matrix the same size as X1) +smooth2(X2) = dot( X2, X2 ) + dot( s2, X2 ) + 4.5 ("s2" is a matrix the same size as X2) + +and the non-smooth/constraint terms are: + +g_1(z) = indicator set of the positive orthant of R^10 +g_2(z) = ||z||_2 (usual Euclidean norm) in R^15 +g_3(z) = ||z||_1 (l1 norm) in R^{10 x 20 }. This views a 10 x 20 matrix as a 200 x 1 vector. +g_4(z) = indicator set of positive orthant of R^{20 x 22 }, i.e. each element must be >= 0 coordinate-wise + +The affine operators are picked arbitrary, but some of them (#3 and #4) have a range that +is a set of matrices, rather than a set of vectors, in order to make +this more interesting. + +%} + +fileName = fullfile('reference_solutions','complicatedProblem1'); +randn('state',29324); +rand('state',9332); +% rng(3481); % this only works on new versions of Matlab + +% -- Variables -- +% Two sets of variables, X1 (a matrix of size n1 x n2 ) and X2 (matrix of N1 x N2 ) +n1 = 10; n2 = 20; +N1 = 12; N2 = 18; +X1 = zeros( n1, n2 ); +X2 = zeros( N1, N2 ); + +% -- the smooth terms -- +% Note: the inner product used is the matrix inner product +% that induces the Frobenius norm. +s1 = randn(n1,n2); +smooth1 = smooth_linear( s1, 3.4 ); +s2 = randn(N1,N2); +smooth2 = smooth_quad( 1, s2, 4.5 ); +% and the same thing, but in a format that CVX likes: +smoothF = @(X1,X2) vec(X1)'*vec(s1) + 3.4 + vec(X2)'*vec(s2) + 4.5 + ... + sum_square(vec(X2))/2; + +% -- some proximal terms -- + +nProx = 4; +prox1 = proj_Rplus; +prox2 = proj_l2; % primal is norm( , 2) +prox3 = proj_linf(1); % this can take matrix varaibles... +prox4 = proj_Rplus; % this can take matrix variables... +% Sizes of proxes (i.e. sizes of dual variables, i.e. size of range of linear terms) +d1 = [ 10, 1 ]; +d2 = [ 15, 1 ]; +d3 = [ n1, n2 ]; +d4 = [ 20, 22 ]; + + +% -- and linear terms -- + +% for prox1: +const1 = randn(d1); +temp1 = randn( prod(d1), n1*n2); +A1_X1 = linop_compose( linop_matrix(temp1), linop_vec([n1,n2]) ); +temp2 = randn( prod(d1), N1*N2); +A1_X2 = linop_compose( linop_matrix(temp2), linop_vec([N1,N2]) ); + +A1 = @(X1,X2) temp1*vec(X1) + temp2*vec(X2) + const1; + +% for prox2: (matrix variable) +const2 = randn(d2); +temp1 = randn( prod(d2), n1*n2); +A2_X1 = linop_compose( linop_matrix(temp1), linop_vec([n1,n2]) ); +temp2 = randn( prod(d2), N1*N2); +A2_X2 = linop_compose( linop_matrix(temp2), linop_vec([N1,N2]) ); + +A2 = @(X1,X2) temp1*vec(X1) + temp2*vec(X2) + const2; + +% for prox3: +const3 = 0; +A3_X1 = 63.4; % this represents abstract scaling, i.e. any size input +A3_X2 = 0; % this reprsents the zero linear operator + +A3 = @(X1,X2) A3_X1*X1; + +% for prox4: (matrix variable) +const4 = randn(d4); +temp1 = randn( prod(d4), n1*n2); +A4_X1 = linop_compose( linop_matrix(temp1), linop_vec([n1,n2]) ); +rs = linop_adjoint( linop_vec(d4) ); +A4_X1 = linop_compose( rs, A4_X1 ); +mat1 = @(x) reshape( x, d4(1), d4(2) ); + +temp2 = randn( prod(d4), N1*N2); +A4_X2 = linop_compose( linop_matrix(temp2), linop_vec([N1,N2]) ); +rs = linop_adjoint( linop_vec(d4) ); +A4_X2 = linop_compose( rs, A4_X2 ); + +A4 = @(X1,X2) mat1( temp1*vec(X1) + temp2*vec(X2) ) + const4; + +% -- set the smoothing parameter -- +mu = 1; + +if exist([fileName,'.mat'],'file') + load(fileName); % contains X_CVX + fprintf('Loaded problem from %s\n', fileName ); +else + % Get reference solution in CVX + % First, get a solution to the smoothed version: + cvx_begin + variables X1(n1,n2) X2(N1,N2) + % it's important that we use norm(vec(...),1) and not just norm(...,1), + % otherwise CVX interprets this with the wrong implicit inner product + minimize( smoothF( X1, X2 ) + ... + norm( A2(X1,X2), 2 ) + norm( vec(A3(X1,X2)) , 1 ) + ... + mu*( sum_square(vec(X1)) + sum_square(vec(X2)) )/2 ) + subject to + A1(X1,X2) >= 0 % constraint is dual of prox1 + A4(X1,X2) >= 0 % constraint is dual of prox2 + cvx_end + X_CVX_smoothed{1} = X1; + X_CVX_smoothed{2} = X2; + + % Second, get a solution to the unsmoothed version + cvx_begin + variables X1(n1,n2) X2(N1,N2) + minimize( smoothF( X1, X2 ) + ... + norm( A2(X1,X2), 2 ) + norm( vec(A3(X1,X2)) , 1 ) ) + subject to + A1(X1,X2) >= 0 + A4(X1,X2) >= 0 + cvx_end + X_CVX{1} = X1; + X_CVX{2} = X2; + + save(fileName,'X_CVX', 'X_CVX_smoothed'); + fprintf('Saved data to file %s\n', fileName); +end + +% Verify constraint are satisfied +fprintf('Is A1(x1,x2) >= 0? The min element is %g\n', min( A1(X_CVX{1},X_CVX{2} )) ) +fprintf('Is A2(x1,x2) >= 0? The min element is %g\n', min(min(A4(X_CVX{1},X_CVX{2}))) ) +%% before running TFOCS, scale the problem perhaps? +% This is one way to see how big the norms are: +% nrm11 = linop_test( A1_X1 ); % 15.7 +% nrm12 = linop_test( A1_X2 ); % 16.6 +% +% nrm21 = linop_test( A2_X1 ); % 17.8 +% nrm22 = linop_test( A2_X2 ); % 18.6 +% +% nrm31 = linop_test( A3_X1 ); % 63.4 +% nrm32 = linop_test( A3_X2 ); % 0 +% +% nrm41 = linop_test( A4_X1 ); % 35.6 +% nrm42 = linop_test( A4_X2 ); % 34.6 + +%% now, run TFOCS. First, try it without continuation + +x0 = []; +z0 = []; +opts = struct('continuation',false,'maxits',1500,'debug',true); % using 'debug' mode to print out useful information +opts.printEvery = 50; + +% Pick a scaling factor "s" (optional: set to "1" to have no effect) +s = .5; % helps a little bit +% (note that if we scale prox3 by "s", then we multiply the corresponding +% affine part by "s", rather than divide them by "s", since prox3 +% is really for the dual and not the primal). + +mu = 1; +opts.errFcn{1} = @(f,d,p) norm( p{1}-X_CVX_smoothed{1}); % compare to smoothed reference solution +opts.errFcn{2} = @(f,d,p) norm( p{2}-X_CVX_smoothed{2}); + +[xAll,outParam,optsOut] = tfocs_SCD( {smooth1,smooth2}, ... + { A1_X1, A1_X2, const1; A2_X1, A2_X2, const2; ... + A3_X1*s, A3_X2*s, const3*s; A4_X1, A4_X2, const4 }, ... + {prox1,prox2,prox_scale(prox3,s),prox4},... + mu, x0, z0, opts ); + +mnConstraint1 = min(min( A1(xAll{1},xAll{2} ) ) ); +mnConstraint2 = min(min( A4(xAll{1},xAll{2} ) ) ); +fprintf('First constraint violated by: %g\n', mnConstraint1); +fprintf('Second constraint violated by: %g\n', mnConstraint2 ); + +% Check that we are within acceptable limits +er = sqrt(norm( xAll{1} - X_CVX_smoothed{1} )^2 + norm( xAll{2} - X_CVX_smoothed{2} )^2 )/... + sqrt( norm(X_CVX_smoothed{1})^2 + norm(X_CVX_smoothed{2})^2 ); % should be about .006 + +if er > 0.04 || mnConstraint1 < -.01 || mnConstraint2 < -.01 + error('Failed the test'); +else + disp('This test successfully passed'); +end + +%% run TFOCS with continuation +x0 = []; +z0 = []; +% type "tfocs" at the command to see possible options +opts = struct('continuation',true,'maxits',1500); +opts.printEvery = 50; +opts.tol = 1e-4; +opts.stopCrit = 4; +opts.printStopCrit = true; + +% type "continuation" at the command to see possible options +contOpts = struct( 'maxIts', 8 , 'muDecrement', 0.8 ); +% ask for increased accuracy on the final solve +contOpts.finalTol = 1e-5; + +s = .5; % helps a little bit +% (note that if we scale prox3 by "s", then we multiply the corresponding +% affine part by "s", rather than divide them by "s", since prox3 +% is really for the dual and not the primal). + +mu = 10; +opts.errFcn{1} = @(f,d,p) norm( p{1}-X_CVX{1}); % compare to unsmoothed reference solution +opts.errFcn{2} = @(f,d,p) norm( p{2}-X_CVX{2}); + +[xAll,outParam,optsOut] = tfocs_SCD( {smooth1,smooth2}, ... + { A1_X1, A1_X2, const1; A2_X1, A2_X2, const2; ... + A3_X1*s, A3_X2*s, const3*s; A4_X1, A4_X2, const4 }, ... + {prox1,prox2,prox_scale(prox3,s),prox4},... + mu, x0, z0, opts, contOpts ); + +mnConstraint1 = min(min( A1(xAll{1},xAll{2} ) ) ); +mnConstraint2 = min(min( A4(xAll{1},xAll{2} ) ) ); +fprintf('First constraint violated by: %g\n', mnConstraint1); +fprintf('Second constraint violated by: %g\n', mnConstraint2 ); + +% Check that we are within acceptable limits +er = sqrt(norm( xAll{1} - X_CVX{1} )^2 + norm( xAll{2} - X_CVX{2} )^2 )/... + sqrt( norm(X_CVX{1})^2 + norm(X_CVX{2})^2 ); % should be about .006 + +if er > 0.4 || mnConstraint1 < -.01 || mnConstraint2 < -.01 + error('Failed the test'); +else + disp('This test successfully passed'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. \ No newline at end of file diff --git a/examples/smallscale/test_nuclearNorm.m b/examples/smallscale/test_nuclearNorm.m new file mode 100644 index 0000000..cd8da2f --- /dev/null +++ b/examples/smallscale/test_nuclearNorm.m @@ -0,0 +1,111 @@ +%{ + Tests nuclear norm minimization, + + min_X ||X||_* +s.t. + P(X) == b, or || P(X) - b || <= eps + +where X is a M x N matrix, and P is an observation operator + +The solvers solve a regularized version, using +||X||_* + mu/2*||X-X_0||_F^2 +You can turn on the continuation feature to reduce the effect of nonzero mu + +%} +nuclear_norm = @(x) sum(svd(x,'econ')); + +% Try to load the problem from disk +fileName = fullfile('reference_solutions','nuclearNorm_problem1_noiseless'); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + + % Generate a new problem + + randn('state',sum('NuclearNorm')); + rand('state',sum('NuclearNorm2')); + + M = 30; N = 30; R = 2; + + df = R*(M+N-R); + oversample = 5; + Left = randn(M,R); + Right = randn(N,R); + k = round(oversample*df); + k = min( k, round(.8*M*N) ); + omega = randperm(M*N); + omega = sort(omega(1:k)).'; + + X_original = Left*Right'; % the "original" signal -- may not be optimal value + b_original = X_original(omega); + EPS = 0; % noise level + b = b_original; + obj_original = nuclear_norm(X_original); + + % get reference via CVX + cvx_begin + cvx_precision best + variable Xcvx(M,N) + minimize norm_nuc(Xcvx) + subject to + Xcvx(omega) == b + cvx_end + X_reference = Xcvx; % the nuclear norm minimizer + obj_reference = nuclear_norm(X_reference); + + save(fileName,'X_original','X_reference','omega','b','obj_original',... + 'Left','Right','EPS','b_original','R','obj_reference'); + fprintf('Saved data to file %s\n', fileName); + +end + +[M,N] = size(X_reference); +norm_X_reference = norm(X_reference,'fro'); +er_reference = @(x) norm(x-X_reference,'fro')/norm_X_reference; +resid = @(x) norm(x(omega)-b)/norm(b); % change if b is noisy + +[omegaI,omegaJ] = ind2sub([M,N],omega); +mat = @(x) reshape(x,M,N); +vec = @(x) x(:); + +k = length(omega); +p = k/(M*N); +df = R*(M+N-R); +fprintf('%d x %d rank %d matrix, observe %d = %.1f x df = %.1f%% entries\n',... + M,N,R,k,k/df,p*100); +fprintf(' Nuclear norm solution and original matrix differ by %.2e\n',... + norm(X_reference-X_original,'fro')/norm_X_reference ); + +%% set some parameters + +testOptions = {}; +opts = []; +opts.alg = 'GRA'; +opts.errFcn = @(f,dual,x) norm( x - X_reference,'fro')/norm(X_reference,'fro'); + +mu = .001; +X0 = 0; + +% The Lipschitz bound is L = 1/mu +% opts.L0 = 1/mu; +[ x, out, opts ] = solver_sNuclearBP( {M,N,omega}, b, mu, X0, [], opts ); +% Check that we are within allowable bounds +if out.err(end) < 1e-5 + disp('Everything is working'); +else + error('Failed the test'); +end +%% test noisy version (just pretend there's a tiny amount of noise) +disp('== Noisy variant =='); +epsilon = 1e-6; % if there really were noise, this should be an estimate of the std +[ x, out, opts ] = solver_sNuclearBPDN( {M,N,omega}, b, epsilon, mu, X0, [], opts ); +% Check that we are within allowable bounds +if out.err(end) < 1e-5 + disp('Everything is working'); +else + error('Failed the test'); +end +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_psdCompletion.m b/examples/smallscale/test_psdCompletion.m new file mode 100644 index 0000000..672973b --- /dev/null +++ b/examples/smallscale/test_psdCompletion.m @@ -0,0 +1,136 @@ +%{ +Test the trace-constrained least-squares problem with positive +semi-definite matrix variables + + minimize (1/2)*norm( A * X - b )^2 + with the constraint that X is positive semi-definite ( X >= 0 ) +and optionally, + the constraint trace(X) <= lambda + +%} + +% Try to load the problem from disk +fileName = fullfile('reference_solutions','traceLS_problem2_noisy'); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + % Generate a new problem + randn('state',sum('Trace')+5); + rand('state',sum('Trace2')+5); + + N = 30; R = 2; + + df = R*N; + oversample = 5; + Left = randn(M,R); + Right = Left; + k = round(oversample*df); + k = min( k, round(.8*N*N) ); + omega = randperm(N*N); + omega = sort(omega(1:k)).'; + + X_original = Left*Right'; % the "original" signal -- may not be optimal value + b_original = X_original(omega); + EPS = .001; % noise level + noise = EPS * randn(k,1); % this destroys symmetry... + b = b_original + noise; + lambda = trace( X_original ); + objective = @(X) sum_square( X(omega) - b )/2; + obj_original = objective(X_original); + + % get references via CVX + cvx_begin + cvx_precision best + cvx_quiet true + variable Xcvx(N,N) + minimize objective(Xcvx) + subject to + Xcvx == semidefinite(N) + cvx_end + X_reference_noTraceConstraint = Xcvx; + fprintf('Difference between convex solution (no trace constraint) and original signal: %.2e\n', ... + norm( Xcvx - X_original, 'fro' ) ); + + cvx_begin + cvx_precision best + cvx_quiet true + variable Xcvx(N,N) + minimize objective(Xcvx) + subject to + Xcvx == semidefinite(N) + trace(Xcvx) <= lambda + cvx_end + X_reference = Xcvx; % the trace norm minimizer + obj_reference = objective(X_reference); + fprintf('Difference between convex solution (with trace constraint) and original signal: %.2e\n', ... + norm( Xcvx - X_original, 'fro' ) ); + save(fileName,'X_original','X_reference','X_reference_noTraceConstraint',... + 'omega','b','obj_original',... + 'Left','EPS','b_original','R','obj_reference','lambda'); + fprintf('Saved data to file %s\n', fileName); + +end + +[M,N] = size(X_reference); +norm_X_reference = norm(X_reference,'fro'); +er_reference = @(x) norm(x-X_reference,'fro')/norm_X_reference; +norm_X_reference2= norm(X_reference_noTraceConstraint,'fro'); +er_reference2 = @(x) norm(x-X_reference_noTraceConstraint,'fro')/norm_X_reference2; +objective = @(X) norm( X(omega) - b )^2/2; + +[omegaI,omegaJ] = ind2sub([M,N],omega); +mat = @(x) reshape(x,M,N); +vec = @(x) x(:); + +k = length(omega); +p = k/(M*N); +df = R*(M+N-R); +fprintf('%d x %d rank %d matrix, observe %d = %.1f x df = %.1f%% entries\n',... + M,N,R,k,k/df,p*100); +fprintf(' Trace norm solution and original matrix differ by %.2e\n',... + norm(X_reference-X_original,'fro')/norm_X_reference ); +%% Solve unconstrained version. No smoothing is necessary! +opts = struct('maxIts',500); +opts.errFcn{1} = @(f,x) er_reference2(x); % no trace constraint +opts.errFcn{2} = @(f,x) sum( abs(eig(x)) > 1e-5 ); % numerical rank +% tell it to use eigs instead of eig +% opts.largescale = true; % ( but not recommended) + +% opts.symmetrize = true; % another option + +A = sparse( omegaI,omegaJ,b,N,N); +[x,out] = solver_psdComp( A, opts ); +% Note: we do not expect to get zero error because there might be more +% than one optimal solution for this problem (since there are not +% so many constraints) +% Check that we have a feasible solution +fprintf('Objective is %.2e, min eigenvalue is %.2e\n', objective(x),... + min(eig(x)) ); +%% Solve trace constrained version. No smoothing necessary! +opts = struct('maxIts',1500,'tol',1e-9); +opts.errFcn{1} = @(f,x) er_reference(x); % no trace constraint +opts.errFcn{2} = @(f,x) trace(x)-lambda; +opts.errFcn{3} = @(f,x) sum( abs(eig(x)) > 1e-5 ); % numerical rank +% we can also symmetrize "omega", but it makes little difference, +% and gives slightly differen value than CVX, since CVX symmetrizes +% differently: +% opts.symmetrize = true; +A = sparse( omegaI,omegaJ,b,N,N); +[x,out] = solver_psdCompConstrainedTrace( A,lambda, opts ); + +h=figure(); +semilogy(out.err(:,1)); + +% Check that we are within allowable bounds +if out.err(end,1) < 1e-5 + disp('Everything is working'); +else + error('Failed the test'); +end + +%% +close(h) +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_quadratic.m b/examples/smallscale/test_quadratic.m new file mode 100644 index 0000000..46356d7 --- /dev/null +++ b/examples/smallscale/test_quadratic.m @@ -0,0 +1,44 @@ +%% Tests the solvers on a simple unconstrained quadratic function + +%{ + Solve: + + minimize_x c'x + x'Dx/2 + + as an example of using TFOCS without the "SCD" interface + +%} + +randn( 'state', sum('quadratic test') ); +N = 100; +c = randn(N,1); +D = randn(N,N); +D = D * D' + .5*eye(N); +x_star = - D \ c; % the true answer +x0 = zeros(N,1); + +% Here's what you could do... +% f = @(x) c'*x + x'*D*x/2; +% grad_f = @(x) c + D*x; +% smoothF = @(x) wrapper_objective( f, grad_f, x ); + +% Here's a simpler way: +smoothF = smooth_quad(D,c); + +x_error = @(x) norm(x-x_star,Inf); + +opts = []; +opts.errFcn = { @(f,x) x_error(x)}; +opts.restart = 100; + +[ x, out, optsOut ] = tfocs( smoothF, [], [],x0, opts ); +% Check that we are within allowable bounds +if out.err(end) < 1e-5 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_quadratic_constrained.m b/examples/smallscale/test_quadratic_constrained.m new file mode 100644 index 0000000..817dd7c --- /dev/null +++ b/examples/smallscale/test_quadratic_constrained.m @@ -0,0 +1,83 @@ +function test_quadratic_constrained +%% Tests the solvers on a simple constrained quadratic function +%{ + Solve: + + minimize_x c'x + x'Dx/2 + subject to ||x||_1 <= 10 + + as an example of using TFOCS without the "SCD" interface + (since the objective is smooth and we can project, there's + no need to smooth and solve the dual) + + It's also an example of using 2 objective functions + +%} + +%% Now, add in constraints + +randn( 'state', sum('quadratic test') ); +N = 100; + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% CONSTRUCT THE TEST PROBLEM % +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +c = randn(N,1); +D = randn(N,N); +D = D * D' + .5*eye(N); +s = svd(D); +% x_star = - D \ c; +% L = max(s); +% mu = min(s); +% if mu < eps, disp('WARNING: may have 0 eigenvalues'); end +% f_star = 0.5 * c' * x_star; +% n_star = norm( x_star ); +x0 = zeros(N,1); + +% f = @(x) c'*x + x'*D*x/2; +% grad_f = @(x) c + D*x; +% smoothF = @(x) wrapper_objective( f, grad_f, x ); +% -- or -- +% smoothF = smooth_quad(D,c); +% -- or (demonstrating how to use 2 objective functions) -- +f1 = @(x) c'*x; +f2 = @(x) x'*D*x/2; +g1 = @(x) c; +g2 = @(x) D*x; +smoothF = { @(x) wrapper_objective(f1,g1,x); @(x) wrapper_objective(f2,g2,x) }; +linearF = { 1 ; 1 }; + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% SET UP THE TEST PARAMETERS % +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +opts = []; +opts.tol = 1e-16; +opts.maxits = 3000; +opts.restart = 100; + +%%%%%%%%%%%%%%%%%%%%% +% ADD IN CONSTRAINT % +%%%%%%%%%%%%%%%%%%%%% +% projectorF = proj_simplex(10); % x >=0, sum(x) = 10 +projectorF = proj_l1(10); % l1 ball, radius 10 (e.g. norm(x,1) <= 10) + + +%%%%%%%%%%%%%%%%% +% RUN THE TESTS % +%%%%%%%%%%%%%%%%% +[ x, out, optsOut ] = tfocs( smoothF, linearF, projectorF,x0, opts ); +% Check that we are within allowable bounds + + + +function [v,gr] = wrapper_objective(f,g,x) +v = f(x); +if nargout > 1 + gr = g(x); +end +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_sBP.m b/examples/smallscale/test_sBP.m new file mode 100644 index 0000000..b5c3ec9 --- /dev/null +++ b/examples/smallscale/test_sBP.m @@ -0,0 +1,144 @@ +%{ + Tests basis pursuit + + min_x ||X||_1 +s.t. + A(X) == b + +The solvers solve a regularized version, using +||x||_1 + mu/2*||x-x_0||_2^2 + +This problem is a linear program, so if mu is sufficiently small, +the smoothing has exactly no effect. + +As a reference solution, we use either "x_ref" which is the interior-point +method solution, or we use "x_original", which is the original signal +used to generate the measurements b. In general, "x_original" is not +the solution to the minimization problem; but under certain circumstances, +such as when "x_original" is very sparse, it is the solution to the minimization problem. +This is the fundamental result from compressed sensing. +When this situation applies, "x_original" is the correct solution to machine precision, +and will be more accurate than even "x_ref". + +%} + +% Before running this, please add the TFOCS base directory to your path + +% Try to load the problem from disk +fileName = fullfile('reference_solutions','basispursuit_problem1_smoothed_noiseless'); +randn('state',34324); + +% We don't want to store "A" +rand('state',34324); +N = 1024; +M = round(N/2); +K = round(M/5); +A = randn(M,N); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + % Generate a new problem + x = zeros(N,1); + T = randsample(N,K); + x(T) = randn(K,1); + + b = A*x; + EPS = 0; + b_original = b; + x_original = x; + + mu = .01*norm(x,Inf); + x0 = zeros(N,1); + % Note: with equality constraints, this is an LP, so for mu small + % enough (but > 0), we have exact relaxation. + + % get reference via CVX + tic + cvx_begin + cvx_precision best + variable xcvx(N,1) + minimize norm(xcvx,1) + mu/2*sum_square(xcvx-x0) + subject to + A*xcvx == b + cvx_end + time_IPM = toc; + x_ref = xcvx; + obj_ref = norm(x_ref,1) + mu/2*sum_square(x_ref-x0); + + save(fileName,'x_ref','b','x_original','mu',... + 'EPS','b_original','obj_ref','x0','time_IPM'); + fprintf('Saved data to file %s\n', fileName); + +end + + +[M,N] = size(A); +K = nnz(x_original); +norm_x_ref = norm(x_ref); +norm_x_orig = norm(x_original); +er_ref = @(x) norm(x-x_ref)/norm_x_ref; +er_signal = @(x) norm(x-x_original)/norm_x_orig; +resid = @(x) norm(A*x-b)/norm(b); % change if b is noisy + +fprintf('\tA is %d x %d, original signal has %d nonzeros\n', M, N, K ); +fprintf('\tl1-norm solution and original signal differ by %.2e (mu = %.2e)\n', ... + norm(x_ref - x_original)/norm(x_original),mu ); + +%% Call the TFOCS solver +% er = er_ref; % error with reference solution (from IPM) +er = er_signal; % error from original signal +% obj_ref = norm(x_ref,1) + mu/2*norm(x_ref-x0)^2; +obj_ref = norm(x_original,1) + mu/2*norm(x_original-x0)^2; +opts = []; +opts.errFcn = { @(f,dual,primal) er(primal), ... + @(f,dual,primal) obj_ref - f }; +z0 = []; % we don't have a good guess for the dual +tic; +[ x, out, optsOut ] = solver_sBP( A, b, mu, x0, z0, opts ); +time_TFOCS = toc; + +fprintf('--------------------------------------------------------\n'); +fprintf('Results:\n-original signal-\t-IPM solution- -TFOCS solution-\n'); +fprintf('---------------------+-----------------+----------------\n'); +fprintf('Number of nonzeros:\n\t%d\t\t%d\t\t%d\n',... + nnz(x_original),nnz(x_ref), nnz(x) ); +fprintf('Error vs. original, rel. l2 norm:\n\tN/A\t\t%.2e\t%.2e\n',... + er_signal(x_ref), er_signal(x) ); +er_signal1 = @(x) norm(x-x_original,Inf); +fprintf('Error vs. original, lInf norm:\n\tN/A\t\t%.2e\t%.2e\n',... + er_signal1(x_ref), er_signal1(x) ); +fprintf('Time to solve:\n\tN/A\t\t%.1fs\t\t%.1fs\n',... + time_IPM, time_TFOCS ); +fprintf('--------------------------------------------------------\n'); + +% Check that we are within allowable bounds +if out.err(end,1) < 1e-8 + disp('Everything is working'); +else + error('Failed the test'); +end +%% Here is how you can view the error history on a graph +figure(); +semilogy( out.err(:,1) ); +xlabel('iterations'); ylabel('error'); + +%% Here are some alternative ways to call it +opts = []; +opts.maxIts = 500; + +A_TFOCS = linop_matrix( A ); % not necessary, but one way to do it +[ x, out, optsOut ] = solver_sBP( A_TFOCS, b, mu, x0, z0, opts ); + +% We can also pass in function handles +Af = @(x) A*x; +At = @(y) A'*y; +A_TFOCS = linop_handles( [M,N], Af, At ); +[ x, out, optsOut ] = solver_sBP( A_TFOCS, b, mu, x0, z0, opts ); + +%% close plots +close all + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_sBPDN.m b/examples/smallscale/test_sBPDN.m new file mode 100644 index 0000000..a6b3849 --- /dev/null +++ b/examples/smallscale/test_sBPDN.m @@ -0,0 +1,102 @@ +%{ + Tests basis pursuit de-noising + + min_x ||x||_1 +s.t. + || A(x) - b || <= eps + +The solvers solve a regularized version, using + ||x||_1 + mu/2*||x-x_0||_2^2 + +see also test_sBP.m + +%} + +% Before running this, please add the TFOCS base directory to your path +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); +%% +% Try to load the problem from disk +fileName = fullfile('reference_solutions','basispursuit_problem1_smoothed_noisy'); +randn('state',34324); +rand('state',34324); +N = 1024; +M = round(N/2); +K = round(M/5); +A = randn(M,N); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + + % Generate a new problem + x = zeros(N,1); + T = randsample(N,K); + x(T) = randn(K,1); + + b_original = A*x; + snr = 30; % SNR in dB + b = myAwgn(b_original,snr); + EPS = norm(b-b_original); + x_original = x; + + mu = .01*norm(x,Inf); + x0 = zeros(N,1); + + % get reference via CVX + tic + cvx_begin + cvx_precision best + variable xcvx(N,1) + minimize norm(xcvx,1) + mu/2*sum_square(xcvx-x0) + subject to + norm(A*xcvx - b ) <= EPS + cvx_end + time_IPM = toc; + x_ref = xcvx; + obj_ref = norm(x_ref,1) + mu/2*sum_square(x_ref-x0); + + save(fileName,'x_ref','b','x_original','mu',... + 'EPS','b_original','obj_ref','x0','time_IPM','snr'); + fprintf('Saved data to file %s\n', fileName); + +end + + +[M,N] = size(A); +K = nnz(x_original); +norm_x_ref = norm(x_ref); +norm_x_orig = norm(x_original); +er_ref = @(x) norm(x-x_ref)/norm_x_ref; +er_signal = @(x) norm(x-x_original)/norm_x_orig; +resid = @(x) norm(A*x-b)/norm(b); % change if b is noisy + +fprintf('\tA is %d x %d, original signal has %d nonzeros\n', M, N, K ); +fprintf('\tl1-norm solution and original signal differ by %.2e (mu = %.2e)\n', ... + norm(x_ref - x_original)/norm(x_original),mu ); + +%% Call the TFOCS solver +er = er_ref; % error with reference solution (from IPM) +opts = []; +opts.restart = 500; +opts.errFcn = { @(f,dual,primal) er(primal), ... + @(f,dual,primal) obj_ref - f }; +opts.maxIts = 1000; +z0 = []; % we don't have a good guess for the dual +tic; +[ x, out, optsOut ] = solver_sBPDN( A, b, EPS, mu, x0, z0, opts ); +time_TFOCS = toc; + +fprintf('Solution has %d nonzeros. Error vs. IPM solution is %.2e\n',... + nnz(x), er(x) ); + +% Check that we are within allowable bounds +if out.err(end,1) < 1e-6 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_sBPDN_W.m b/examples/smallscale/test_sBPDN_W.m new file mode 100644 index 0000000..840d859 --- /dev/null +++ b/examples/smallscale/test_sBPDN_W.m @@ -0,0 +1,149 @@ +%{ + Tests the "analysis" form of basis pursuit de-noising + + min_x ||Wx||_1 +s.t. + || A(x) - b || <= eps + +The solvers solve a regularized version, using + ||Wx||_1 + mu/2*||x-x_0||_2^2 + +see also test_sBP.m and test_sBPDN.m + +%} + +% Before running this, please add the TFOCS base directory to your path +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); + +% Try to load the problem from disk +fileName = fullfile('reference_solutions','basispursuit_W_problem1_smoothed_noisy'); +randn('state',34324); +rand('state',34324); + +N = 512; +M = round(N/2); +K = round(M/5); + +A = randn(M,N); +x = zeros(N,1); + +% introduce a sparsifying transform "W" +d = round(4*N); % redundant +Wf = @(x) dct(x,d); % zero-padded DCT +downsample = @(x) x(1:N,:); +Wt = @(y) downsample( idct(y) ); % transpose of W +W = Wf(eye(N)); % make an explicit matrix for CVX +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); + +else + + % Generate a new problem + + % the signal x consists of several pure tones at random frequencies + for k = 1:K + x = x + randn()*sin( rand()*pi*(1:N) + 2*pi*rand() ).'; + end + + + b_original = A*x; + snr = 40; % SNR in dB + b = myAwgn(b_original,snr); + EPS = norm(b-b_original); + x_original = x; + + mu = .01*norm(Wf(x),Inf); + x0 = zeros(N,1); + + % get reference via CVX + tic + cvx_begin + cvx_precision best + variable xcvx(N,1) + minimize norm(W*xcvx,1) + mu/2*sum_square(xcvx-x0) + subject to + norm(A*xcvx - b ) <= EPS + cvx_end + time_IPM = toc; + x_ref = xcvx; + obj_ref = norm(W*x_ref,1) + mu/2*sum_square(x_ref-x0); + + save(fileName,'x_ref','b','x_original','mu',... + 'EPS','b_original','obj_ref','x0','time_IPM','snr','d'); + fprintf('Saved data to file %s\n', fileName); + +end + + +[M,N] = size(A); +norm_x_ref = norm(x_ref); +norm_x_orig = norm(x_original); +er_ref = @(x) norm(x-x_ref)/norm_x_ref; +er_signal = @(x) norm(x-x_original)/norm_x_orig; +resid = @(x) norm(A*x-b)/norm(b); % change if b is noisy + +fprintf('\tA is %d x %d\n', M, N ); +fprintf('\tl1-norm solution and original signal differ by %.2e (mu = %.2e)\n', ... + norm(x_ref - x_original)/norm(x_original),mu ); + +%% Call the TFOCS solver +objF = @(x)norm(W*x,1) + mu/2*norm(x-x0).^2; +infeasF = @(x)norm(A*x-b) - EPS; +er = er_ref; % error with reference solution (from IPM) +opts = []; +opts.errFcn = { @(f,dual,primal) er(primal), ... + @(f,dual,primal) obj_ref - f, ... + @(f,dual,primal) infeasF(primal) }; +opts.maxIts = 1000; +opts.tol = 1e-10; +% opts.normA2 = norm(A*A'); +% opts.normW2 = norm(W'*W); +z0 = []; % we don't have a good guess for the dual +tic; +[ x, out, optsOut ] = solver_sBPDN_W( A, W, b, EPS, mu, x0, z0, opts ); +time_TFOCS = toc; +fprintf('x is sub-optimal by %.2e, and infeasible by %.2e\n',... + objF(x) - obj_ref, infeasF(x) ); + +fprintf('Solution has %d nonzeros. Error vs. IPM solution is %.2e\n',... + nnz(x), er(x) ); + +% Check that we are within allowable bounds +if out.err(end,1) < 1e-4 + disp('Everything is working'); +else + error('Failed the test'); +end + + +%% Doing it "by hand" (for debugging), just for 200 iterations +objF = @(x)norm(W*x,1) + mu/2*norm(x-x0).^2; +infeasF = @(x)norm(A*x-b) - EPS; +er = er_ref; % error with reference solution (from IPM) +% er = er_signal; +opts = []; +opts.errFcn = { @(f,dual,primal) er(primal), ... + @(f,dual,primal) obj_ref - f, ... + @(f,dual,primal) infeasF(primal) }; +opts.maxIts = 200; +opts.printEvery = 10; +opts.tol = 1e-10; +z0 = []; +proxScale = sqrt(norm(W'*W)/norm(A*A')); +scale = 1; +prox = { prox_l2( EPS/scale ), proj_linf(proxScale) }; +affineF = {A/scale,-b/scale;W/proxScale,0}; +[ x, out, optsOut ] = tfocs_SCD( [], affineF, prox, mu, x0, z0, opts ); + +% Check that we are within allowable bounds +if out.err(end,1) < 1e-2 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_sBPDN_WW.m b/examples/smallscale/test_sBPDN_WW.m new file mode 100644 index 0000000..b3ce1d4 --- /dev/null +++ b/examples/smallscale/test_sBPDN_WW.m @@ -0,0 +1,130 @@ +%{ + Tests the extended "analysis" form of basis pursuit de-noising + + min_1 alpha*||W_1 x||_1 + beta*|| W_2 x ||_1 +s.t. + || A(x) - b || <= eps + +The solvers solve a regularized version + +see also test_sBP.m and test_sBPDN.m and test_sBPDN_W.m + +%} + +% Before running this, please add the TFOCS base directory to your path +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); + +% Try to load the problem from disk +fileName = fullfile('reference_solutions','basispursuit_WW_problem1_smoothed_noisy'); +randn('state',34324); +rand('state',34324); + +N = 512; +M = round(N/2); +K = round(M/5); + +A = randn(M,N); +x = zeros(N,1); + +% introduce a sparsifying transform "W" +d = round(4*N); % redundant +Wf = @(x) dct(x,d); % zero-padded DCT +downsample = @(x) x(1:N,:); +Wt = @(y) downsample( idct(y) ); % transpose of W +W = Wf(eye(N)); % make an explicit matrix for CVX + +% and add another transform: +d2 = round(2*N); +W2 = randn(d2,N); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + + % Generate a new problem + + alpha = 1; + beta = 2; + + % the signal x consists of several pure tones at random frequencies + for k = 1:K + x = x + randn()*sin( rand()*pi*(1:N) + 2*pi*rand() ).'; + end + + + b_original = A*x; + snr = 40; % SNR in dB + b = myAwgn(b_original,snr); + EPS = norm(b-b_original); + x_original = x; + + mu = .01*norm(Wf(x),Inf); + x0 = zeros(N,1); + + % get reference via CVX + tic + cvx_begin + cvx_precision high + variable xcvx(N,1) + minimize alpha*norm(W*xcvx,1) + beta*norm(W2*xcvx,1) + ... + mu/2*sum_square(xcvx-x0) + subject to + norm(A*xcvx - b ) <= EPS + cvx_end + time_IPM = toc; + x_ref = xcvx; + objF = @(x)alpha*norm(W*x,1) +beta*norm(W2*x,1)+ mu/2*norm(x-x0).^2; + obj_ref = objF(x_ref); + + save(fileName,'x_ref','b','x_original','mu',... + 'EPS','b_original','obj_ref','x0','time_IPM','snr',... + 'd','d2','alpha','beta'); + fprintf('Saved data to file %s\n', fileName); + +end + + +[M,N] = size(A); +norm_x_ref = norm(x_ref); +norm_x_orig = norm(x_original); +er_ref = @(x) norm(x-x_ref)/norm_x_ref; +er_signal = @(x) norm(x-x_original)/norm_x_orig; +resid = @(x) norm(A*x-b)/norm(b); % change if b is noisy + +fprintf('\tA is %d x %d\n', M, N ); +fprintf('\tl1-norm solution and original signal differ by %.2e (mu = %.2e)\n', ... + norm(x_ref - x_original)/norm(x_original),mu ); + +%% Call the TFOCS solver +objF = @(x)alpha*norm(W*x,1) +beta*norm(W2*x,1)+ mu/2*norm(x-x0).^2; +infeasF = @(x)norm(A*x-b) - EPS; +er = er_ref; % error with reference solution (from IPM) +opts = []; +opts.errFcn = { @(f,dual,primal) er(primal), ... + @(f,dual,primal) obj_ref - f, ... + @(f,dual,primal) infeasF(primal) }; +opts.maxIts = 2000; +opts.tol = 1e-10; +% opts.normA2 = norm(A*A'); +% opts.normW2 = norm(W'*W); +z0 = []; % we don't have a good guess for the dual +tic; +[ x, out, optsOut ] = solver_sBPDN_WW( A, alpha,W,beta,W2,b, EPS, mu, x0, z0, opts ); +time_TFOCS = toc; +fprintf('x is sub-optimal by %.2e, and infeasible by %.2e\n',... + objF(x) - obj_ref, infeasF(x) ); + +fprintf('Solution has %d nonzeros. Error vs. IPM solution is %.2e\n',... + nnz(x), er(x) ); + +% Check that we are within allowable bounds +if out.err(end,1) < 1e-3 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_sBPDN_complex.m b/examples/smallscale/test_sBPDN_complex.m new file mode 100644 index 0000000..fe3bb4c --- /dev/null +++ b/examples/smallscale/test_sBPDN_complex.m @@ -0,0 +1,120 @@ +%{ + Tests basis pursuit de-noising + + min_x ||x||_1 +s.t. + || A(x) - b || <= eps + +The solvers solve a regularized version, using + ||x||_1 + mu/2*||x-x_0||_2^2 + +This version allows for complex measurements. + +see also test_sBPDN.m + +%} + +% Before running this, please add the TFOCS base directory to your path +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); + +% Try to load the problem from disk +fileName = fullfile('reference_solutions','basispursuit_problem1_smoothed_noisy_complex'); +randn('state',34324); +rand('state',34324); + +N = 1024; +M = round(N/2); +K = round(M/5); + +A = fft(eye(N)); % A is complex +rowsA = sort(randsample(N,M)); +A = A(rowsA,:); + +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + + % Generate a new problem + + x = zeros(N,1); + T = randsample(N,K); + x(T) = randn(K,1); + + b_original = A*x; + snr = 30; % SNR in dB + b = myAwgn(b_original,snr); + EPS = norm(b-b_original); + x_original = x; + + mu = .01*norm(x,Inf); + x0 = zeros(N,1); + % Note: with equality constraints, this is an LP, so for mu small + % enough (but > 0), we have exact relaxation. + + % get reference via CVX + tic + cvx_begin + cvx_precision best + variable xcvx(N,1) + minimize norm(xcvx,1) + mu/2*sum_square(xcvx-x0) + subject to + norm(A*xcvx - b ) <= EPS + cvx_end + time_IPM = toc; + x_ref = xcvx; + obj_ref = norm(x_ref,1) + mu/2*sum_square(x_ref-x0); + + save(fileName,'x_ref','b','x_original','mu',... + 'EPS','b_original','obj_ref','x0','time_IPM','snr'); + fprintf('Saved data to file %s\n', fileName); + +end + + +[M,N] = size(A); +K = nnz(x_original); +norm_x_ref = norm(x_ref); +norm_x_orig = norm(x_original); +er_ref = @(x) norm(x-x_ref)/norm_x_ref; +er_signal = @(x) norm(x-x_original)/norm_x_orig; +resid = @(x) norm(A*x-b)/norm(b); % change if b is noisy + +fprintf('\tA is %d x %d, original signal has %d nonzeros\n', M, N, K ); +fprintf('\tl1-norm solution and original signal differ by %.2e (mu = %.2e)\n', ... + norm(x_ref - x_original)/norm(x_original),mu ); + +%% Call the TFOCS solver +er = er_ref; % error with reference solution (from IPM) +opts = []; +opts.errFcn = { @(f,dual,primal) er(primal), ... + @(f,dual,primal) obj_ref - f }; +opts.maxIts = 750; + +% Option 1: +mode = 'r2c'; +% mode = 'c2c'; % do NOT use this: it solves a different problem! +AA = A; bb = b; Ahandles = linop_matrix(AA,mode); + +% Option 2: +% AA = [real(A); imag(A)]; bb = [real(b); imag(b)]; +% Ahandles = linop_matrix(AA); + +z0 = []; % we don't have a good guess for the dual +tic; +[ x, out, optsOut ] = solver_sBPDN( Ahandles, bb, EPS, mu, x0, z0, opts ); +time_TFOCS = toc; + +fprintf('Solution has %d nonzeros. Error vs. IPM solution is %.2e\n',... + nnz(x), er(x) ); + +if out.err(end,1) < 1e-3 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_sBPDN_complex_2.m b/examples/smallscale/test_sBPDN_complex_2.m new file mode 100644 index 0000000..5e1cee1 --- /dev/null +++ b/examples/smallscale/test_sBPDN_complex_2.m @@ -0,0 +1,126 @@ +%{ + Tests basis pursuit de-noising + + min_x ||x||_1 +s.t. + || A(x) - b || <= eps + +The solvers solve a regularized version, using + ||x||_1 + mu/2*||x-x_0||_2^2 + +This version allows for complex measurements. + +Unlike test_sBPDN_complex, we also allow x to be complex. +For example, suppose we have a signal f that is the superposition +of a few tones. Then in the "synthesis" formulation, we make the +change-of-variables x = F f, where F is the FFT matrix. +If "PHI" is the measurement matrix, then the linear operator is +A = Phi*F', where F' = inv(F) is the IFFT. + +see also test_sBPDN.m, test_sBPDN_complex, test_sBPDN_W + +%} + +% Before running this, please add the TFOCS base directory to your path +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); + +% Try to load the problem from disk +fileName = fullfile('reference_solutions','basispursuit_problem1_smoothed_noisy_complex_2'); +randn('state',34324); +rand('state',34324); +N = 1024; +M = round(N/2); +M = round(M/5); +Phi = randn(M,N); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); + F = fft(eye(N))/sqrt(N); + A = Phi * F'; +else + + % Generate a new problem + + F = fft(eye(N))/sqrt(N); + A = Phi * F'; + + % introduce a sparsifying transform "W" + f = zeros(N,1); + % the signal x consists of several pure tones at random frequencies + K = 5; % + for k = 1:K + f = f + randn()*sin( rand()*pi*(1:N) + 2*pi*rand() ).'; + end + x = fft(f)/sqrt(N); + + + b_original = A*x; + snr = 30; % SNR in dB + b = myAwgn(b_original,snr); + EPS = norm(b-b_original); + x_original = x; + + mu = .001*norm(x,Inf); + x0 = zeros(N,1); + + % get reference via CVX + tic + cvx_begin + cvx_precision best + variable xcvx(N,1) complex + minimize norm(xcvx,1) + mu/2*sum_square_abs(xcvx-x0) + subject to + norm(A*xcvx - b ) <= EPS + cvx_end + time_IPM = toc; + x_ref = xcvx; + obj_ref = norm(x_ref,1) + mu/2*sum_square_abs(x_ref-x0); + + save(fileName,'x_ref','b','x_original','mu',... + 'EPS','b_original','obj_ref','x0','time_IPM','snr'); + fprintf('Saved data to file %s\n', fileName); + +end + + +[M,N] = size(A); +norm_x_ref = norm(x_ref); +norm_x_orig = norm(x_original); +er_ref = @(x) norm(x-x_ref)/norm_x_ref; +er_signal = @(x) norm(x-x_original)/norm_x_orig; +resid = @(x) norm(A*x-b)/norm(b); % change if b is noisy + +fprintf('\tA is %d x %d, original signal has %d nonzeros\n', M, N, nnz(x_original) ); +fprintf('\tl1-norm solution and original signal differ by %.2e (mu = %.2e)\n', ... + norm(x_ref - x_original)/norm(x_original),mu ); + +%% Call the TFOCS solver +er = er_ref; % error with reference solution (from IPM) +opts = []; +opts.errFcn = { @(f,dual,primal) er(primal), ... + @(f,dual,primal) obj_ref - f }; +opts.maxIts = 4500; +opts.printEvery = 50; +opts.tol = 1e-8; + +mode = 'c2c'; +AA = A; bb = b; Ahandles = linop_matrix(AA,mode); + +z0 = []; % we don't have a good guess for the dual +tic; +[ x, out, optsOut ] = solver_sBPDN( Ahandles, bb, EPS, mu, x0, z0, opts ); +time_TFOCS = toc; + +fprintf('Solution has %d nonzeros. Error vs. IPM solution is %.2e\n',... + nnz(x), er(x) ); + +if out.err(end,1) < 1e-5 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_sBPDN_nonnegative.m b/examples/smallscale/test_sBPDN_nonnegative.m new file mode 100644 index 0000000..5e5e8eb --- /dev/null +++ b/examples/smallscale/test_sBPDN_nonnegative.m @@ -0,0 +1,136 @@ +%{ + Tests non-negative basis pursuit + + min_x ||X||_1 +s.t. + || A(X) - b|| <= eps and x >= 0 + +The solvers solve a regularized version, using +||x||_1 + mu/2*||x-x_0||_2^2 + + +See also test_sBPDN.m and test_sBP_nonnegative.m + +%} + +% Before running this, please add the TFOCS base directory to your path + +% Try to load the problem from disk +fileName = fullfile('reference_solutions','basispursuit_problem1_smoothed_noisy_nonnegative'); +randn('state',25442); +rand('state',2452); + +N = 1024; +M = round(N/2); +K = round(M/2); + +A = randn(M,N); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + + % Generate a new problem + + x = zeros(N,1); + T = randsample(N,K); + x(T) = .1+rand(K,1); % x >= 0 + + b = A*x; + noise = .1*norm(b)*randn(M,1)/sqrt(M); + EPS = .9*norm(noise); + b_original = b + noise; + x_original = x; + + mu = .01*norm(x,Inf); + x0 = zeros(N,1); + + % get reference via CVX + tic + cvx_begin + cvx_precision best + variable xcvx(N,1) + minimize norm(xcvx,1) + mu/2*sum_square(xcvx-x0) + subject to + norm( A*xcvx - b ) <= EPS + xcvx >= 0 + cvx_end + time_IPM = toc; + x_ref = xcvx; + obj_ref = norm(x_ref,1) + mu/2*sum_square(x_ref-x0); + + % get reference via CVX, this time, without the x >= 0 constraing + tic + cvx_begin + cvx_precision best + variable xcvx(N,1) + minimize norm(xcvx,1) + mu/2*sum_square(xcvx-x0) + subject to + norm( A*xcvx - b ) <= EPS + cvx_end + time_IPM = toc; + x_pureBP = xcvx; + obj_ref = norm(x_ref,1) + mu/2*sum_square(x_ref-x0); + + save(fileName,'x_ref','b','x_original','mu',... + 'EPS','b_original','obj_ref','x0','time_IPM','x_pureBP'); + fprintf('Saved data to file %s\n', fileName); + +end + + +[M,N] = size(A); +K = nnz(x_original); +norm_x_ref = norm(x_ref); +norm_x_orig = norm(x_original); +norm_x_pureBP = norm(x_pureBP); +er_ref = @(x) norm(x-x_ref)/norm_x_ref; % err w.r.t. CVX +er_pureBP = @(x) norm(x-x_pureBP)/norm_x_pureBP; % err w.r.t. CVX w.o. nonneg constraint +er_signal = @(x) norm(x-x_original)/norm_x_orig; % err w.r.t. signal +resid = @(x) norm(A*x-b)/norm(b); % change if b is noisy + +fprintf('\tA is %d x %d, original signal has %d nonzeros\n', M, N, K ); +fprintf('\tl1-norm solution and original signal differ by:\t\t\t\t%.2e (mu = %.2e)\n', ... + norm(x_pureBP - x_original)/norm(x_original),mu ); +fprintf('\tl1-norm solution, with x >=0 constraint, and original signal differ by:\t%.2e (mu = %.2e)\n', ... + norm(x_ref - x_original)/norm(x_original),mu ); + +disp('Note: because of the noise, we do not expect to get zero error'); + +%% Call the TFOCS solver +er = er_ref; % error with reference solution (from CVX) +opts = []; +% To see all possible options, run "tfocs()" +opts.errFcn = { @(f,dual,primal) er(primal), ... + @(f,dual,primal) obj_ref - f, ... + @(f,dual,primal) nnz(primal) }; +opts.maxIts = 400; +z0 = []; % we don't have a good guess for the dual + +opts.nonneg = true; % tell it to add the x >= 0 constraint + +tic; +x = solver_sBPDN( A, b, EPS, mu, x0, z0, opts ); +time_TFOCS = toc; + +fprintf('for (original signal, IPM solution, TFOCS solution),\n NNZ:\n\t%d\t\t%d\t\t%d\n',... + nnz(x_original),nnz(x_ref), nnz(x) ); +fprintf(' error vs. original, rel. l2 norm:\n\t%.2e\t%.2e\t%.2e\n',... + 0, er_signal(x_ref), er_signal(x) ); +er_signal1 = @(x) norm(x-x_original,Inf); +fprintf(' error vs. original, lInf norm:\n\t%.2e\t%.2e\t%.2e\n',... + 0, er_signal1(x_ref), er_signal1(x) ); +fprintf(' time to solve:\n\tN/A\t\t%.1fs\t\t%.1fs\n',... + time_IPM, time_TFOCS ); + +% Check that we are within allowable bounds +if er(x) < .01 + disp('Everything is working'); +else + error('Failed the test'); +end + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_sBPDN_withContinuation.m b/examples/smallscale/test_sBPDN_withContinuation.m new file mode 100644 index 0000000..e94a465 --- /dev/null +++ b/examples/smallscale/test_sBPDN_withContinuation.m @@ -0,0 +1,147 @@ +%{ + Tests basis pursuit de-noising + + min_x ||x||_1 +s.t. + || A(x) - b || <= eps + +The solvers solve a regularized version, using + ||x||_1 + mu/2*||x-x_0||_2^2 + +in this file, we will use continuation to eliminate +the effect of the "mu" term. + +see also test_sBPDN.m + +%} + +% Before running this, please add the TFOCS base directory to your path + +% Try to load the problem from disk +fileName = fullfile('reference_solutions','basispursuit_problem1_noisy'); +randn('state',34324); +rand('state',34324); +N = 1024; +M = round(N/2); +K = round(M/5); +A = randn(M,N); +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + + % Generate a new problem + x = zeros(N,1); + T = randsample(N,K); + x(T) = randn(K,1); + + b_original = A*x; + snr = 30; % SNR in dB + b = myAwgn(b_original,snr); + EPS = norm(b-b_original); + x_original = x; + + mu = .01*norm(x,Inf); + x0 = zeros(N,1); + + % get reference via CVX + tic + cvx_begin + cvx_precision best + variable xcvx(N,1) + minimize norm(xcvx,1) + subject to + norm(A*xcvx - b ) <= EPS + cvx_end + time_IPM = toc; + x_ref = xcvx; + obj_ref = norm(x_ref,1); + + save(fileName,'x_ref','b','x_original','mu',... + 'EPS','b_original','obj_ref','x0','time_IPM','snr'); + fprintf('Saved data to file %s\n', fileName); + +end + + +[M,N] = size(A); +K = nnz(x_original); +norm_x_ref = norm(x_ref); +norm_x_orig = norm(x_original); +er_ref = @(x) norm(x-x_ref)/norm_x_ref; +er_signal = @(x) norm(x-x_original)/norm_x_orig; +resid = @(x) norm(A*x-b)/norm(b); % change if b is noisy + +fprintf('\tA is %d x %d, original signal has %d nonzeros\n', M, N, K ); +fprintf('\tl1-norm solution and original signal differ by %.2e (mu = %.2e)\n', ... + norm(x_ref - x_original)/norm(x_original),mu ); + +%% Call the TFOCS solver +er = er_ref; % error with reference solution (from IPM) +opts = []; +opts.restart = 500; +opts.errFcn = { @(f,dual,primal) er(primal) }; +opts.maxIts = 1000; +opts.countOps = true; +% To see more possible options, run the command "tfocs" + +z0 = []; % we don't have a good guess for the dual +tic; + + +% -- with continuation: +opts.continuation = true; + +% when using continuation, a good stopping criteria is this one: +opts.stopCrit = 4; opts.tol = 1e-6; +opts.printStopcrit = 1; % this will display the value used in the stopping criteria calculation +[ x, out, optsOut ] = solver_sBPDN( A, b, EPS, mu, x0, z0, opts); + + + +time_TFOCS = toc; + +fprintf('Solution has %d nonzeros. Error vs. IPM solution is %.2e\n',... + nnz(x), er(x) ); +% Test we are within bounds: +if out.err(end,1) < 1e-3 + disp('Everything is working'); +else + error('Failed the test'); +end +%% Advanced options for continuation +% To see all possible options, run "continuation" with no inputs: +continuation() + +continuationOptions = []; + +% for example, we can make sure that we only take at most 200 +% iterations (except for the very last outer iteration): +continuationOptions.innerMaxIts = 200; + +% and instead of just 3 iterations (the default), let's do 5 iterations: +continuationOptions.maxIts = 5; + +% and decrease the tolerance of the outer loop +continuationOptions.tol = 1e-5; + +opts.printEvery = Inf; % to suppress output from inner iterations +continuationOptions.verbose = true; % "true" by default + +% if you want mu to decrease: +continuationOptions.muDecrement = 0.8; + +[ x, out, optsOut ] = solver_sBPDN( A, b, EPS, mu, x0, z0, opts, continuationOptions); + +% Test we are within bounds: +if out.err(end,1) < 1e-3 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_sBP_nonnegative.m b/examples/smallscale/test_sBP_nonnegative.m new file mode 100644 index 0000000..663c58d --- /dev/null +++ b/examples/smallscale/test_sBP_nonnegative.m @@ -0,0 +1,149 @@ +%{ + Tests non-negative basis pursuit + + min_x ||X||_1 +s.t. + A(X) == b and x >= 0 + +The solvers solve a regularized version, using +||x||_1 + mu/2*||x-x_0||_2^2 + +%} + +% Before running this, please add the TFOCS base directory to your path + +% Try to load the problem from disk +fileName = fullfile('reference_solutions','basispursuit_problem1_smoothed_noiseless_nonnegative'); +randn('state',34324); +rand('state',34324); + +N = 1024; +M = round(N/2); +K = round(M/2); + +A = randn(M,N); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + + % Generate a new problem + + x = zeros(N,1); + T = randsample(N,K); + x(T) = .1+rand(K,1); + + b = A*x; + EPS = 0; + b_original = b; + x_original = x; + + mu = .01*norm(x,Inf); + x0 = zeros(N,1); + % Note: with equality constraints, this is an LP, so for mu small + % enough (but > 0), we have exact relaxation. + + % get reference via CVX + tic + cvx_begin + cvx_precision best + variable xcvx(N,1) + minimize norm(xcvx,1) + mu/2*sum_square(xcvx-x0) + subject to + A*xcvx == b + xcvx >= 0 + cvx_end + time_IPM = toc; + x_ref = xcvx; + obj_ref = norm(x_ref,1) + mu/2*sum_square(x_ref-x0); + + % get reference via CVX, this time, without the x >= 0 constraing + tic + cvx_begin + cvx_precision best + variable xcvx(N,1) + minimize norm(xcvx,1) + mu/2*sum_square(xcvx-x0) + subject to + A*xcvx == b + cvx_end + time_IPM = toc; + x_pureBP = xcvx; + obj_ref = norm(x_ref,1) + mu/2*sum_square(x_ref-x0); + + save(fileName,'x_ref','b','x_original','mu',... + 'EPS','b_original','obj_ref','x0','time_IPM','x_pureBP'); + fprintf('Saved data to file %s\n', fileName); + +end + + +[M,N] = size(A); +K = nnz(x_original); +norm_x_ref = norm(x_ref); +norm_x_orig = norm(x_original); +norm_x_pureBP = norm(x_pureBP); +er_ref = @(x) norm(x-x_ref)/norm_x_ref; +er_pureBP = @(x) norm(x-x_pureBP)/norm_x_pureBP; +er_signal = @(x) norm(x-x_original)/norm_x_orig; +resid = @(x) norm(A*x-b)/norm(b); % change if b is noisy + +fprintf('\tA is %d x %d, original signal has %d nonzeros\n', M, N, K ); +fprintf('\tl1-norm solution and original signal differ by:\t\t\t\t%.2e (mu = %.2e)\n', ... + norm(x_pureBP - x_original)/norm(x_original),mu ); +fprintf('\tl1-norm solution, with x >=0 constraint, and original signal differ by:\t%.2e (mu = %.2e)\n', ... + norm(x_ref - x_original)/norm(x_original),mu ); + +%% Call the TFOCS solver +er = er_ref; % error with reference solution (from IPM) +% er = er_signal; % error from original signal +% er = er_pureBP; % the "pureBP" solution did not have x>=0 +opts = []; +opts.errFcn = { @(f,dual,primal) er(primal), ... + @(f,dual,primal) obj_ref - f, ... + @(f,dual,primal) nnz(primal) }; +opts.restart = 1000; +z0 = []; % we don't have a good guess for the dual +tic; + +projectionF = {proj_Rn; proj_Rplus }; + +scaleA = 1/norm(A); +[x,out,optsOut] = tfocs_SCD( prox_l1, { linop_compose(A,scaleA), -b*scaleA; 1,0 }, projectionF, mu, x0, z0, opts ); + + + +time_TFOCS = toc; + +fprintf('for (original signal, IPM solution, TFOCS solution),\n NNZ:\n\t%d\t\t%d\t\t%d\n',... + nnz(x_original),nnz(x_ref), nnz(x) ); +fprintf(' error vs. original, rel. l2 norm:\n\t%.2e\t%.2e\t%.2e\n',... + 0, er_signal(x_ref), er_signal(x) ); +er_signal1 = @(x) norm(x-x_original,Inf); +fprintf(' error vs. original, lInf norm:\n\t%.2e\t%.2e\t%.2e\n',... + 0, er_signal1(x_ref), er_signal1(x) ); +fprintf(' time to solve:\n\tN/A\t\t%.1fs\t\t%.1fs\n',... + time_IPM, time_TFOCS ); + +% Test we are within bounds: +if out.err(end,1) < 1e-6 + disp('Everything is working'); +else + error('Failed the test'); +end +%% As of version 1.0d, the solver_sBP.m file can handle x >= 0 constraints +% This version will be more efficient, since it calls a special +% l1 + non-negative operator, rather than splitting it up. + +opts.nonneg = true; % tell it to add the x >= 0 constraint +[x,out,optsOut] = solver_sBP( A, b, mu, x0, z0, opts ); + +% Test we are within bounds: +if out.err(end,1) < 1e-5 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_sDantzig.m b/examples/smallscale/test_sDantzig.m new file mode 100644 index 0000000..0e26e92 --- /dev/null +++ b/examples/smallscale/test_sDantzig.m @@ -0,0 +1,147 @@ +%{ + Tests the Dantzig Selector + + min_x ||x||_1 +s.t. + || D*A'*(A*x - b) || <= delta + +The solvers solve a regularized version, using + ||x||_1 + mu/2*||x-x_0||_2^2 + +see also test_sBPDN.m + +%} + +% Before running this, please add the TFOCS base directory to your path +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); + +% Try to load the problem from disk +fileName = fullfile('reference_solutions','dantzig_problem1_smoothed_noisy'); +randn('state',34324); +rand('state',34324); +N = 1024; +M = round(N/2); +K = round(M/5); +A = randn(M,N); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + + % Generate a new problem + + normA2 = norm( (A'*A) )^2; + x = zeros(N,1); + T = randsample(N,K); + x(T) = randn(K,1); + + + b_original = A*x; + snr = 30; % SNR in dB + b = myAwgn(b_original,snr); + z = b - b_original; + sigma = std(z); % estimate of standard deviation + + % compute D and delta so that original signal is + % feasible with probability 1 - alpha + alpha = 0.1; + Anorms = sqrt(sum(A.^2))'; + nTrials = min(4*N,400); + w = randn(M,nTrials); + supAtz = sort(max( (A'*w) ./Anorms(:,ones(1,nTrials)))); + thresh = supAtz(round(nTrials*(1-alpha))); % empirical + d = thresh*sigma*Anorms; % a vector, for Dantzig solvers + if all(d > 1e-10) + delta = mean(d); + D = delta./d; % watch out for division by 0 + else + D = 1; + delta = 0; + end + normA2 = norm( diag(D)*(A'*A) )^2; +% clear d alpha w thresh nTrials + + x_original = x; + + mu = .05*norm(x,Inf); + x0 = zeros(N,1); + % Note:this is an LP, so for mu small + % enough (but > 0), we have exact relaxation. + + % get reference via CVX + tic + cvx_begin + cvx_precision best + variable xcvx(N,1) + minimize norm(xcvx,1) + mu/2*sum_square(xcvx-x0) + subject to + norm(D.*(A'*(A*xcvx - b)),Inf ) <= delta + cvx_end + time_IPM = toc; + x_ref = xcvx; + obj_ref = norm(x_ref,1) + mu/2*sum_square(x_ref-x0); + + save(fileName,'x_ref','b','x_original','mu',... + 'delta','D','b_original','obj_ref','x0','time_IPM','snr'); + fprintf('Saved data to file %s\n', fileName); + +end + + +[M,N] = size(A); +K = nnz(x_original); +norm_x_ref = norm(x_ref); +norm_x_orig = norm(x_original); +er_ref = @(x) norm(x-x_ref)/norm_x_ref; +er_signal = @(x) norm(x-x_original)/norm_x_orig; +resid = @(x) norm(A*x-b)/norm(b); % change if b is noisy + +fprintf('\tA is %d x %d, original signal has %d nonzeros\n', M, N, K ); +fprintf('\tl1-norm solution and original signal differ by %.2e (mu = %.2e)\n', ... + norm(x_ref - x_original)/norm(x_original),mu ); + +%% Call the TFOCS solver +er = er_ref; % error with reference solution (from IPM) +opts = []; +opts.restart = 500; +opts.errFcn = { @(f,dual,primal) er(primal), ... + @(f,dual,primal) obj_ref - f }; +opts.maxIts = 2000; +z0 = []; % we don't have a good guess for the dual +tic; +[ x, out, optsOut ] = solver_sDantzig( {A,D}, b, delta, mu, x0, z0, opts ); +time_TFOCS = toc; + +fprintf('Solution has %d nonzeros. Error vs. IPM solution is %.2e\n',... + nnz(x), er(x) ); + +% Check that we are within allowable bounds +if out.err(end,1) < 1e-4 + disp('Everything is working'); +else + error('Failed the test'); +end +%% plot error +semilogy( out.err(:,1) ) +hold all + +%% Call the solver with W = I +% This is not usually recommended, since it is not necessary +% and creates extra dual variables +opts.restart = 2000; +W = linop_scale(1); % identity +[ x, out, optsOut ] = solver_sDantzig_W( {A,D}, W, b, delta, mu, x0, z0, opts ); +% Check that we are within allowable bounds +if out.err(end,1) < 1e-1 + disp('Everything is working'); +else + error('Failed the test'); +end + +%% close all figures +close all + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_sDantzig_3methods.m b/examples/smallscale/test_sDantzig_3methods.m new file mode 100644 index 0000000..2f23dc5 --- /dev/null +++ b/examples/smallscale/test_sDantzig_3methods.m @@ -0,0 +1,146 @@ +%{ + Tests the Dantzig Selector + + min_x ||x||_1 +s.t. + || D*A'*(A*x - b) || <= delta + +The solvers solve a regularized version, using + ||x||_1 + mu/2*||x-x_0||_2^2 + +see also test_sDantzig.m + +This demo shows three formulations of the Dantzig selector +Instead of calling a pre-built solver, we show how to call +tfocs_SCD directly. + +%} + +% Before running this, please add the TFOCS base directory to your path + +% Try to load the problem from disk +mu = 0; +fileName = fullfile('reference_solutions','dantzig_problem1_smoothed_noisy'); +randn('state',34324); +rand('state',34324); +N = 1024; +M = round(N/2); +K = round(M/5); +A = randn(M,N); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + disp('Please run test_sDantzig.m to setup the file'); +end + + +[M,N] = size(A); +K = nnz(x_original); +norm_x_ref = norm(x_ref); +norm_x_orig = norm(x_original); +er_ref = @(x) norm(x-x_ref)/norm_x_ref; +er_signal = @(x) norm(x-x_original)/norm_x_orig; +resid = @(x) norm(A*x-b)/norm(b); % change if b is noisy + +fprintf('\tA is %d x %d, original signal has %d nonzeros\n', M, N, K ); +fprintf('\tl1-norm solution and original signal differ by %.2e (mu = %.2e)\n', ... + norm(x_ref - x_original)/norm(x_original),mu ); + +%% Call the TFOCS solver +er = er_ref; % error with reference solution (from IPM) +opts = []; +opts.restart = 500; +opts.errFcn = { @(f,dual,primal) er(primal), ... + @(f,dual,primal) obj_ref - f }; +opts.maxIts = 1000; +opts.printEvery = 100; +z0 = []; % we don't have a good guess for the dual + + +%% Method 1: use the epigraph of the l_infinity norm as the cone +% Note: this is equivalent to calling: +% [ x, out, optsOut ] = solver_sDantzig( {A,D}, b, delta, mu, x0, z0, opts ); + +DAtb = D.*(A'*b); +DD = @(x) D.*(x); +objectiveF = prox_l1; +affineF = {linop_matrix(diag(D)*(A'*A)), -DAtb }; +dualproxF = prox_l1( delta ); +[ x, out, opts ] = tfocs_SCD( objectiveF, affineF, dualproxF, mu, x0, z0, opts ); +x1 = x; +out1 = out; + +fprintf('Solution has %d nonzeros. Error vs. IPM solution is %.2e\n',... + nnz(x), er(x) ); + +% Check that we are within allowable bounds +if out.err(end,1) < 1e-3 + disp('Everything is working'); +else + error('Failed the test'); +end +%% Method 2: use the LP formulation +% Instead of the constraint ||Ax-b||_infty <= delta +% (where "A" is really DA'A), +% think of it as +% -(Ax-b) + delta >= 0 +% (Ax-b) + delta >= 0 + +objectiveF = prox_l1; +affineF = {linop_matrix(-diag(D)*(A'*A)), DAtb + delta;... + linop_matrix( diag(D)*(A'*A)), -DAtb + delta; }; + +dualproxF = { proj_Rplus; proj_Rplus }; +[ x, out, opts ] = tfocs_SCD( objectiveF, affineF, dualproxF, mu, x0, z0, opts ); +x2 = x; +out2 = out; + +% Check that we are within allowable bounds +if out.err(end,1) < 1e-3 + disp('Everything is working'); +else + error('Failed the test'); +end +%% Method 3: put objective into constraint +% This is the trick we do with solver_sDantzig_W, to deal with ||Wx||_1 +% Instead of minimizing ||x||_1, we minimize t, +% with the constraint that ||x||_1 <= t +% This version has some scaling considerations -- if we +% are careful about scaling, the dual problem will be solved much faster. + +% Note: we still have to deal with the original constraints. We +% can use either method 1 or method 2 above. Here, we'll use +% method 1. + + +objectiveF = []; % tfocs_SCD recognizes this special objective +normA2 = norm( diag(D)*A'*A )^2; +scale = 1/sqrt(normA2); +affineF = {linop_matrix(diag(D)*(A'*A)), -DAtb; linop_scale(1/scale), 0 }; +dualproxF = { prox_l1(delta); proj_linf(scale) }; +[ x, out, opts ] = tfocs_SCD( objectiveF, affineF, dualproxF, mu, x0, z0, opts ); +x3 = x; +out3 = out; + +% Check that we are within allowable bounds +if out.err(end,1) < 1e-2 + disp('Everything is working'); +else + error('Failed the test'); +end +%% plot +figure; +semilogy( out1.err(:,1) ); +hold all +semilogy( out2.err(:,1) ); +semilogy( out3.err(:,1) ); +legend('Method 1 (epigraph cone)','Method 2 (LP)','Method 3 (W=I)' ); + +%% close all plots +close all + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. \ No newline at end of file diff --git a/examples/smallscale/test_sTV.m b/examples/smallscale/test_sTV.m new file mode 100644 index 0000000..3d70ca5 --- /dev/null +++ b/examples/smallscale/test_sTV.m @@ -0,0 +1,118 @@ +%{ + Tests total-variation problem + + min_x ||x||_TV +s.t. + || A(x) - b || <= eps + +The solvers solve a regularized version + +see also test_sBP.m and test_sBPDN_W.m + +requires the image processing toolbox for this demo +(but the TFOCS solver does not rely on this toolbox) + +See also the TV examples in examples/largescale + +%} + +% Before running this, please add the TFOCS base directory to your path + +myAwgn = @(x,snr) x + ... + 10^( (10*log10(sum(abs(x(:)).^2)/length(x(:))) - snr)/20 )*randn(size(x)); + +% Try to load the problem from disk +fileName = fullfile('reference_solutions','tv_problem1_smoothed_noisy'); +randn('state',245); +rand('state',245); +n = 32; +n1 = n; +n2 = n-1; % testing the code with non-square signals +N = n1*n2; +M = round(N/2); +A = randn(M,N); +if exist([fileName,'.mat'],'file') + load(fileName); + fprintf('Loaded problem from %s\n', fileName ); +else + + % Generate a new problem + + n = max(n1,n2); + x = phantom(n); + x = x(1:n1,1:n2); + x_original = x; + + mat = @(x) reshape(x,n1,n2); + vec = @(x) x(:); + + + b_original = A*vec(x_original); + snr = 40; % SNR in dB + b = myAwgn(b_original,snr); + EPS = norm(b-b_original); + + tv = linop_TV( [n1,n2], [], 'cvx' ); + + mu = .005*norm( tv(x_original) ,Inf); + x0 = zeros(n1,n2); + + % get reference via CVX + tic + cvx_begin + cvx_precision best + variable xcvx(n1,n2) + minimize tv(xcvx) + mu/2*sum_square(vec(xcvx)-vec(x0) ) + subject to + norm(A*vec(xcvx) - b ) <= EPS + cvx_end + time_IPM = toc; + x_ref = xcvx; + obj_ref = tv(x_ref) + mu/2*sum_square(vec(x_ref)-vec(x0) ); + + save(fileName,'x_ref','b','x_original','mu',... + 'EPS','b_original','obj_ref','x0','time_IPM','snr'); + fprintf('Saved data to file %s\n', fileName); + +end + +imshow( [x_original, x_ref] ); + +[M,N] = size(A); +[n1,n2] = size(x_original); +norm_x_ref = norm(x_ref,'fro'); +norm_x_orig = norm(x_original,'fro'); +er_ref = @(x) norm(vec(x)-vec(x_ref))/norm_x_ref; +er_signal = @(x) norm(x-x_original)/norm_x_orig; +resid = @(x) norm(A*vec(x)-b)/norm(b); % change if b is noisy + + +%% Call the TFOCS solver +er = er_ref; % error with reference solution (from IPM) +opts = []; +opts.restart = 1000; +opts.errFcn = { @(f,dual,primal) er(primal), ... + @(f,dual,primal) obj_ref - f }; +opts.maxIts = 1000; + +W = linop_TV( [n1,n2] ); +normW = linop_TV( [n1,n2], [], 'norm' ); +opts.normW2 = normW^2; +z0 = []; % we don't have a good guess for the dual +tic; +[ x, out, optsOut ] = solver_sBPDN_W( A, W, b, EPS, mu, vec(x0), z0, opts ); +time_TFOCS = toc; + +fprintf('Solution has %d nonzeros. Error vs. IPM solution is %.2e\n',... + nnz(x), er(x) ); + +% Check that we are within allowable bounds +if out.err(end,1) < 1e-4 + disp('Everything is working'); +else + error('Failed the test'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/examples/smallscale/test_variousSolvers.m b/examples/smallscale/test_variousSolvers.m new file mode 100644 index 0000000..ba11ae8 --- /dev/null +++ b/examples/smallscale/test_variousSolvers.m @@ -0,0 +1,70 @@ +%{ +Solves a simple quadratic problem +(with optimal solution known in closed-form) +with all the various solvers + + +the quadratic problem is: + +min_x c'x + x'Px/2, P > 0, hence the optimal solution is x = -inv(Q)*c + +On a simple quadratic example like this, when there is no projection, +all the solver behave similarly. So this script is more useful for debugging +to make sure that all solvers are working + +%} + +%% Make a problem +N = 100; +randn('state',2334); +P = randn(N); P = P*P' + .1*eye(N); % ensure P is symmetric positive definite +c = randn(N,1); + +x_ref = -P\c; + +%% Solve with TFOCS +f = smooth_quad( P, c ); +affine = []; +projector = []; +x0 = zeros(N,1); +opts = []; +opts.printEvery = 500; +opts.tol = 1e-8; +opts.printStopCrit = true; +opts.restart = 350; % use this since the problem is strongly convex +opts.errFcn = @(f,x) norm( x-x_ref)/norm(x_ref); +opts.maxIts = 750; + +% The 'N83' and 'AT' methods can take advantage when you know +% the exact value of the strong convexity parameter +% (and the Lipschitz constant). In this case, do not +% use restart. But we won't test this right now, +% since it is unusual that you have information +% about the strong convexity constant. +% opts.restart = Inf; +% opts.mu = min(eig(P)); % strong convexity parameter. N83 can make use of this +% opts.Lexact = max(eig(P)); + +solverList = { 'GRA', 'AT', 'LLM', 'N07', 'N83', 'TS' }; +% solverList = { 'GRA', 'AT', 'N83', 'TS' }; +% solverList = { 'LLM','N07'}; % these take two steps per iteration + +figure(1); clf; + +for k = 1:length( solverList ) + solver = solverList{k}; + opts.alg = solver; + fprintf('\nSolver: %s\n\n', solver ); + [x,out,optsOut] = tfocs( f, affine, projector, x0, opts ); + semilogy( out.err ); + hold all +end +legend( solverList ); + + +%% close all figures +close all + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. \ No newline at end of file diff --git a/linop_TV.m b/linop_TV.m new file mode 100644 index 0000000..77a1bf8 --- /dev/null +++ b/linop_TV.m @@ -0,0 +1,199 @@ +function op = linop_TV( sz, variation, action ) + +%LINOP_TV 2D Total-Variation (TV) linear operator. +% OP = LINOP_TV( SZ ) returns a handle to a TFOCS linear operator that +% implements the total variation linear operator on an M x N grid; +% that is, to be applied to matrices of size [M,N]. +% By default, it expects to operate on M*N x 1 vectors +% but if SZ = {M,N}, then expects to operate on M x N matrices +% +% N.B. In this form, OP does not calculate the TV norm but rather +% a vector of size M*N x 1 such that TV(X) = norm( OP(X,1), 1 ). +% +% TV = LINOP_TV( X ) returns ||X||_TV if X is bigger than 2 x 2 +% +% OP = LINOP_TV( SZ, VARIANT ) +% if VARIANT is 'regular' (default), +% assumes zeros on the boundary (Dirichlet boundary conditions) +% if VARIANT is 'circular', +% assumes circularity, so x(:,N+1) - x(:,N) +% is calculated by x(:,1) - x(:,N) (and similarly +% for row differences) [when SZ={M,N}] +% +% [...] = LINOP_TV(SZ, VARIATION, ACTION ) +% if ACTION is 'handle', returns a TFOCS function handle (default) +% if ACTION is 'cvx', returns a function handle suitable for CVX +% if ACTION is 'matrix', returns the explicit TV matrix +% (real part corresponds to horizontal differences, +% imaginary part correspond to vertical differences) +% if ACTION is 'norm', returns an estimate of the norm +% +% TODO: check circular case for non-square domains +% + +error(nargchk(1,3,nargin)); +if nargin < 2 || isempty(variation), variation = 'regular'; end +if nargin < 3 || isempty(action), action = 'handle'; end + +CALCULATE_TV = false; +if numel(sz) > 4 + CALCULATE_TV = true; + X = sz; + sz = size(X); +end + +if iscell(sz) + n1 = sz{1}; + n2 = sz{2}; +else + n1 = sz(1); + n2 = sz(2); +end + +% Setup the Total-Variation operators +mat = @(x) reshape(x,n1,n2); + +if strcmpi(action,'matrix') || strcmpi(action,'cvx') + switch lower(variation) + case 'regular' + e = ones(max(n1,n2),1); + e2 = e; + e2(n2:end) = 0; + J = spdiags([-e2,e], 0:1,n2,n2); + I = eye(n1); + Dh = kron(J,I); % horizontal differences, sparse matrix + % see also blkdiag + + e2 = e; + e2(n1:end) = 0; + J = spdiags([-e2,e], 0:1,n1,n1); + I = eye(n2); + Dv = kron(I,J); % vertical differences, sparse matrix + case 'circular' + e = ones(max(n1,n2),1); + e2 = e; +% e2(n2:end) = 0; + J = spdiags([-e2,e], 0:1,n2,n2); + J(end,1) = 1; + I = eye(n1); + Dh = kron(J,I); % horizontal differences, sparse matrix + % see also blkdiag + + e = ones(max(n1,n2),1); + e2 = e; +% e2(n1:end) = 0; + J = spdiags([-e2,e], 0:1,n1,n1); + J(end,1) = 1; + I = eye(n2); + Dv = kron(I,J); % vertical differences, sparse matrix + end + if strcmpi(action,'matrix') + op = Dh + 1i*Dv; + else + % "norms" is a CVX function, but we can over-load it (see sub-function below) + op = @(X) sum( norms( [Dh*X(:), Dv*X(:)]' ) ); + end + return; +end + +switch lower(variation) + case 'regular' + Dh = @(X) vec( [diff(X,1,2), zeros(n1,1)] ); + diff_h = @(X) [zeros(n1,1),X(:,1:end-1)] - [X(:,1:end-1),zeros(n1,1) ]; + Dv = @(X) vec( [diff(X,1,1); zeros(1,n2)] ); + diff_v = @(X) [zeros(1,n2);X(1:end-1,:)] - [X(1:end-1,:);zeros(1,n2) ]; + % sometimes diff_v is much slower than diff_h + % We can exploit data locality by working with transposes + diff_v_t = @(Xt) ([zeros(n2,1),Xt(:,1:end-1)] - [Xt(:,1:end-1),zeros(n2,1)])'; + case 'circular' + % For circular version, 2 x 2 case is special. +% error('not yet implemented'); + Dh = @(X) vec( [diff(X,1,2), X(:,1) - X(:,end)] ); + % diff_h needs to be checked + diff_h = @(X) [X(:,end),X(:,1:end-1)] - X; + % diff_v needs to be checked + Dv = @(X) vec( [diff(X,1,1); X(1,:) - X(end,:) ] ); + diff_v = @(X) [X(end,:);X(1:end-1,:)] - X; + diff_v_t = @(Xt) ([Xt(:,end),Xt(:,1:end-1)] - Xt)'; + otherwise + error('Bad variation parameter'); +end +if iscell(sz) + Dh_transpose = @(X) diff_h(mat(X)) ; +% Dv_transpose = @(X) diff_v(mat(X)) ; + Dv_transpose = @(X) diff_v_t(mat(X)') ; % faster +else + Dh_transpose = @(X) vec( diff_h(mat(X)) ); +% Dv_transpose = @(X) vec( diff_v(mat(X)) ); + Dv_transpose = @(X) vec( diff_v_t(mat(X)') ); % faster +end + +TV = @(x) ( Dh(mat(x)) + 1i*Dv(mat(x)) ); % real to complex +TVt = @(z) ( Dh_transpose(real(z)) + Dv_transpose(imag(z)) ); + +if CALCULATE_TV + op = norm( TV(X), 1 ); + return; +end + +if strcmpi(action,'norm') + % to compute max eigenvalue, I use a vector + % that is very likely to be the max eigenvector: + % matrix with every entry alternating -1 and 1 + even = @(n) ~( n - 2*round(n/2) ); % returns 1 if even, 0 if odd + Y = zeros( n1 + even(n1), n2 + even(n2) ); + nn = numel(Y); + Y(:) = (-1).^(1:nn); + Y = Y(1:n1,1:n2); + op = norm( TV(Y) )/norm(Y(:)); + + % Nearly equivalent to: + % norm(full( [real(tv); imag(tv)] ) ) + % where tv is the matrix form +else + if iscell(sz) + szW = { [n1,n2], [n1*n2,1] }; + else + szW = sz(1)*sz(2); % n1 * n2 + szW = [szW,szW]; + end + op = @(x,mode)linop_tv_r2c(szW,TV,TVt,x,mode); +end + +function y = linop_tv_r2c( sz, TV, TVt, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = TV( realcheck( x ) ); + case 2, y = realcheck( TVt( x ) ); +end + +function y = realcheck( y ) +if ~isreal( y ), + error( 'Unexpected complex value in linear operation.' ); +end + + +function y = norms( x, p, dim ) +sx = size(x); +if nargin < 3, dim = 1; end +if nargin < 2 || isempty(p), p = 2; end +if isempty(x) || dim > length(sx) || sx(dim)==1 + p = 1; +end +switch p, + case 1, + y = sum( abs( x ), dim ); + case 2, + y = sqrt( sum( x .* conj( x ), dim ) ); + case Inf, + y = max( abs( x ), [], dim ); + case {'Inf','inf'} + y = max( abs( x ), [], dim ); + otherwise, + y = sum( abs( x ) .^ p, dim ) .^ ( 1 / p ); +end + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/linop_TV3D.m b/linop_TV3D.m new file mode 100644 index 0000000..1fcc1fe --- /dev/null +++ b/linop_TV3D.m @@ -0,0 +1,197 @@ +function op = linop_TV3D( sz, variation, action ) + +%LINOP_TV3D 3D Total-Variation (TV) linear operator. +% OP = LINOP_TV3D( SZ ) returns a handle to a TFOCS linear operator that +% implements the total variation linear operator on an M x N x P grid; +% that is, to be applied to volume stacks of size [M,N,P]. +% By default, it expects to operate on M*N*P x 1 vectors +% but if SZ = {M,N,P}, then expects to operate on M x N x P matrices +% +% TV = LINOP_TV3D( X ) returns ||X||_TV if X is bigger than 2 x 2 x 2 +% +% OP = LINOP_TV3D( SZ, VARIANT ) +% if VARIANT is 'regular' (default), +% ... TODO +% if VARIANT is 'circular', +% ... TODO +% +% [...] = LINOP_TV3D(SZ, VARIATION, ACTION ) +% if ACTION is 'handle', returns a TFOCS function handle (default) +% if ACTION is 'cvx', returns a function handle suitable for CVX +% if ACTION is 'matrix', returns the explicit TV matrix +% (real part corresponds to horizontal differences, +% imaginary part correspond to vertical differences) +% if ACTION is 'norm', returns an estimate of the norm +% +% Contributed by Mahdi Hosseini (mahdi.hosseini@mail.utoronto.ca) +% Has not been extensively tested + +error(nargchk(1,3,nargin)); +if nargin < 2 || isempty(variation), variation = 'regular'; end +if nargin < 3 || isempty(action), action = 'handle'; end + +CALCULATE_TV = false; +if numel(sz) > 6 + CALCULATE_TV = true; + X = sz; + sz = size(X); +end +nDim = numel(sz); + +if iscell(sz) + n1 = sz{1}; + n2 = sz{2}; + n3 = sz{3}; +else + n1 = sz(1); + n2 = sz(2); + n3 = sz(3); +end + +% Setup the Total-Variation operators +mat = @(x) reshape(x,n1,n2,n3); + +if strcmpi(action,'matrix') || strcmpi(action,'cvx') + I1 = eye(n1); + I2 = eye(n2); + I3 = eye(n3); + switch lower(variation) + case 'regular' + e = ones(max([n1,n2,n3]),1); + e2 = e; + e2(n2:end) = 0; + J = spdiags([-e2,e], 0:1,n2,n2); + Dh = kron(I3,kron(J,I1)); % horizontal differences, sparse matrix + % see also blkdiag + + e2 = e; + e2(n1:end) = 0; + J = spdiags([-e2,e], 0:1,n1,n1); + Dv = kron(I3,kron(I2,J)); % vertical differences, sparse matrix + + e2 = e; + e2(n3:end) = 0; + J = spdiags([-e2,e], 0:1,n3,n3); + Dd = kron(J,kron(I2,I1)); % Depth differences, sparse matrix + case 'circular' + e = ones(max([n1,n2,n3]),1); + e2 = e; + % e2(n2:end) = 0; + J = spdiags([-e2,e], 0:1,n2,n2); + J(end,1) = 1; + Dh = kron(I3, kron(J,I1)); % horizontal differences, sparse matrix + % see also blkdiag + + e2 = e; + % e2(n1:end) = 0; + J = spdiags([-e2,e], 0:1,n1,n1); + J(end,1) = 1; + Dv = kron(I3, kron(I2,J)); % vertical differences, sparse matrix + + e2 = e; + % e2(n1:end) = 0; + J = spdiags([-e2,e], 0:1,n3,n3); + J(end,1) = 1; + Dd = kron(J, kron(I2,I1)); % vertical differences, sparse matrix + end + if strcmpi(action,'matrix') + op = [Dh;Dv;Dd]; + else + % "norms" is a CVX function + op = @(X) sum( norms( [Dh*X(:), Dv*X(:), Dd*X(:)]' ) ); + end + return; +end + +switch lower(variation) + case 'regular' + Dh = @(X) vec( [diff(X,1,2), zeros(n1,1,n3)] ); + Dv = @(X) vec( [diff(X,1,1); zeros(1,n2,n3)] ); + Dd = @(X) [vec(diff(X,1,3)); vec(zeros(n1,n2,1))]; + + diff_h = @(X) [zeros(n1,1,n3),X(:,1:end-1,:)] - ... + [X(:,1:end-1,:),zeros(n1,1,n3)]; + diff_v = @(X) [zeros(1,n2,n3);X(1:end-1,:,:)] - ... + [X(1:end-1,:,:);zeros(1,n2,n3)]; + diff_d = @(X) mat([vec(zeros(n1,n2,1));vec(X(:,:,1:end-1))] - ... + [vec(X(:,:,1:end-1));vec(zeros(n1,n2,1))]); + case 'circular' + % For circular version, 2 x 2 case is special. + % error('not yet implemented'); + Dh = @(X) vec( [diff(X,1,2), X(:,1,:) - X(:,end,:)] ); + Dv = @(X) vec( [diff(X,1,1); X(1,:,:) - X(end,:,:)] ); + Dd = @(X) [vec(diff(X,1,3)); vec(X(:,:,1) - X(:,:,end))]; + % diff_h needs to be checked + diff_h = @(X) [X(:,end,:),X(:,1:end-1,:)] - X; + % diff_v needs to be checked + diff_v = @(X) [X(end,:,:);X(1:end-1,:,:)] - X; + % diff_d needs to be checked + diff_d = @(X) mat([vec(X(:,:,end));vec(X(:,:,1:end-1))]) - X; + otherwise + error('Bad variation parameter'); +end +if iscell(sz) + Dh_transpose = @(X) diff_h(mat(X)) ; + Dv_transpose = @(X) diff_v(mat(X)) ; + Dd_transpose = @(X) diff_d(mat(X)) ; +else + Dh_transpose = @(X) vec( diff_h(mat(X)) ); + Dv_transpose = @(X) vec( diff_v(mat(X)) ); + Dd_transpose = @(X) vec( diff_d(mat(X)) ); +end + +%% TV & TVt Definitions +TV = @(x) [Dh(mat(x)); Dv(mat(x)); Dd(mat(x))]; + +firstThird = @(x) x(1: n1*n2*n3); +secondThird= @(x) x(n1*n2*n3+1: 2*n1*n2*n3); +thirdThird= @(x) x(2*n1*n2*n3+1: 3*n1*n2*n3); +TVt = @(z) ( Dh_transpose(firstThird(z)) +... + Dv_transpose(secondThird(z)) +... + Dd_transpose(thirdThird(z))); + +%% +if CALCULATE_TV + op = norm( TV(X), 1 ); + return; +end + +if strcmpi(action,'norm') + % to compute max eigenvalue, I use a vector + % that is very likely to be the max eigenvector: + % matrix with every entry alternating -1 and 1 + even = @(n) ~( n - 2*round(n/2) ); % returns 1 if even, 0 if odd + Y = zeros( n1 + even(n1), n2 + even(n2), n3 + even(n3) ); + nn = numel(Y); + Y(:) = (-1).^(1:nn); + Y = Y(1:n1,1:n2,1:n3); + op = norm( TV(Y) )/norm(Y(:)); + + % Nearly equivalent to: + % norm(full( [real(tv); imag(tv)] ) ) + % where tv is the matrix form +else + if iscell(sz) + szW = { [n1,n2,n3], [n1*n2*n3,1] }; + else + szW = prod(sz); % n1 * n2 + szW = [nDim*szW, nDim*szW]; + end + op = @(x,mode)linop_tv_r2c(szW,TV,TVt,x,mode); +end + +function y = linop_tv_r2c( sz, TV, TVt, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = TV( realcheck( x ) ); + case 2, y = realcheck( TVt( x ) ); +end + +function y = realcheck( y ) +if ~isreal( y ), + error( 'Unexpected complex value in linear operation.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/linop_adjoint.m b/linop_adjoint.m new file mode 100644 index 0000000..7a2511d --- /dev/null +++ b/linop_adjoint.m @@ -0,0 +1,22 @@ +function op = linop_adjoint( A ) +%LINOP_ADJOINT Computes the adjoint operator of a TFOCS linear operator +%op = LINOP_ADJOINT( A ) +% Returns a function handle to a linear operator that is the adjoint of +% the operator supplied. + +op = @(y,mode)linop_adjoint_impl( A, y, mode ); + +function y = linop_adjoint_impl( A, x, mode ) +switch mode, + case 0, + y = A(x,0); + y([1,2]) = y([2,1]); + case 1, + y = A(x,2); + case 2, + y = A(x,1); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/linop_compose.m b/linop_compose.m new file mode 100644 index 0000000..f050dd8 --- /dev/null +++ b/linop_compose.m @@ -0,0 +1,84 @@ +function op = linop_compose( varargin ) +%LINOP_COMPOSE Composes two TFOCS linear operators +%OP = LINOP_COMPOSE( OP1, OP2, ..., OPN ) +% Constructs a TFOCS-compatible linear operator from the composition of +% two or more linear operators and/or matrices. That is, +% OP(x,1) = OP1(OP2(...(OPN(x,1),...,1),1) +% OP(x,2) = OPN(...(OP2(OP1(x,2),2)...,2) +% If matrices are supplied, they must be real; to include complex +% matrices, convert them first to linear operators using LINOP_MATRIX. + +if nargin == 0, + error( 'Not enough input arguments.' ); +end +sz = { [], [] }; +for k = 1 : nargin, + tL = varargin{k}; + if isempty(tL) || ~isa(tL,'function_handle') && ~isnumeric(tL) || ndims(tL) > 2, + error( 'Arguments must be linear operators, scalars, or matrices.' ); + elseif isnumeric(tL), + if ~isreal(tL), + error( 'S or scalar arguments must be real.' ); + elseif numel(tL) == 1, + tL = linop_scale( tL ); + else + tL = linop_matrix( tL ); + end + varargin{k} = tL; + end + try + tsz = tL([],0); + catch + error( 'Arguments must be linear operators, scalars, or matrices.' ); + end + if isempty(tsz) % i.e. output of linop_identity is [] + tsz = { [], [] }; + end + if isnumeric(tsz), + tsz = { [tsz(2),1], [tsz(1),1] }; + end + + % convert [n1;n2] to [n1,n2] if necessary: + for kk = 1:2 + %if iscolumn( tsz{kk} ) + %tsz{kk} = tsz{kk}.'; + %end + tsz{kk} = tsz{kk}(:).'; + end + + if ~isempty(sz{1}) && ~isempty(tsz{2}) && ~isequal(tsz{2},sz{1}), + for kk = 1:min( numel(tsz{2}), numel( sz{1} ) ) + fprintf('Found incompatible sizes: %d ~= %d\n', tsz{2}(kk), sz{1}(kk) ); + end + error( 'Incompatible dimensions in linear operator composition.' ); + end + if ~isempty(tsz{1}), + sz{1} = tsz{1}; + end + if isempty(sz{2}), + sz{2} = tsz{2}; + end +end +if nargin == 1, + op = varargin{1}; +else + op = @(x,mode)linop_compose_impl( varargin, sz, x, mode ); +end + +function y = linop_compose_impl( ops, sz, y, mode ) +switch mode, + case 0, + y = sz; + case 1, + for k = numel(ops) : -1 : 1, + y = ops{k}( y, 1 ); + end + case 2, + for k = 1 : numel(ops), + y = ops{k}( y, 2 ); + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/linop_dot.m b/linop_dot.m new file mode 100644 index 0000000..4a582ff --- /dev/null +++ b/linop_dot.m @@ -0,0 +1,49 @@ +function op = linop_dot( A, adj ) +%LINOP_DOT Linear operator formed from a dot product. +% OP = LINOP_DOT( A ) returns a handle to a TFOCS linear operator +% whose forward operation is OP(X) = TFOCS_DOT( A, X ). +% OP = LINOP_DOT( A, 1 ) returns the adjoint of that operator. + +switch class( A ), + case 'double', + sz = { size(A), [1,1] }; + case 'cell', + A = tfocs_tuple(A); + sz = { tfocs_size(A), [1,1] }; + case 'tfocs_tuple', + sz = { tfocs_size(A), [1,1] }; + otherwise, + error( 'First input must be a matrix or cell array of matrices.' ); +end +if nargin == 2 && adj, + sz = { sz{2}, sz{1} }; + op = @(x,mode)linop_dot_adjoint( sz, A, x, mode ); +else + op = @(x,mode)linop_dot_forward( sz, A, x, mode ); +end + +function y = linop_dot_forward( sz, A, x, mode ) +switch mode, + case 0, + y = sz; + case 1, + y = tfocs_dot( A, x ); + case 2, + if ~isreal(x), error( 'Unexpected complex input.' ); end + y = A * x; +end + +function y = linop_dot_adjoint( sz, A, x, mode ) +switch mode, + case 0, + y = sz; + case 1, + if ~isreal(x), error( 'Unexpected complex input.' ); end + y = A * x; + case 2, + y = tfocs_dot( A, x ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/linop_fft.m b/linop_fft.m new file mode 100644 index 0000000..61ad9ff --- /dev/null +++ b/linop_fft.m @@ -0,0 +1,192 @@ +function op = linop_fft( N, bigN, cmode, width ) +%LINOP_FFT Fast Fourier transform linear operator. +% OP = LINOP_FFT( N ) +% returns a function handle that computes the 1D FFT. +% If given a matrix, it operates on each column separately (i.e. it +% does NOT automatically switch to a 2D FFT). +% +% By default, it assumes the input the FFT is real, and the output +% is therefore complex and conjugate-symmetric +% +% OP = LINOP_FFT( N, M ) +% will zero-pad the input so that it is size M. Currently, the code +% requires M > N, which is typical in applications. +% The forward linear operator takes a vector of dimension N and returns a +% vector of dimension M; the adjoint takes a M-vector and returns a N-vector. +% By default, M = N. +% +% OP = LINOP_FFT( N, M, CMODE ) +% specifies the complex mode of the linear operator. The choices are: +% 'r2c' (default) : real input and complex (conjugate-symmetric) output +% 'c2c' : complex input and outpuyt +% 'r2r' : real input and real output. The real output +% retains the same number of degrees of freedom because it exploits +% the redudancy in the conjugate-symmetry of the complex output of the fft. +% +% OP = LINOP_FFT( N, M, CMODE, WIDTH ) +% specifies that the domain is the space of N x WIDTH matrices +% (and the range is M x WIDTH ). +% +% new in TFOCS v1.3 + +% Wed, Jan 4 2011 + +error(nargchk(1,4,nargin)); + +% the naming convention "bigN" refers to the fact that we usually +% will pad the input with zeros to take an oversampled FFT. +% In this case, "bigN" is the size of the zero-padded input, +% and hence also the size of the output. +if nargin < 2 || isempty(bigN), bigN = N; end +if bigN < N, error('cannot yet handle bigN < N, but there is no inherent limitiation'); end +if nargin < 3 || isempty(cmode), + cmode = 'r2c'; + warning('TFOCS:FFTdefaultMode','In linop_fft, using default ''r2c'' mode for fft; this will take real-part of input only!'); +end +if nargin < 4 || isempty(width), width = 1; end +sz = { [N,width], [bigN,width] }; + +% allow normalization to make it orthogonal? To do so, divide forward mode by sqrt(bigN) + +switch lower(cmode) + case 'c2c' + op = @(x,mode)linop_fft_c2c( sz, N, bigN, x, mode ); + case 'r2c' + op = @(x,mode)linop_fft_r2c( sz, N, bigN, x, mode ); + case 'r2r' + n2 = bigN/2; + even = (n2 == round(n2) ); % find if n is even or odd + if ~even + n2 = (bigN+1)/2; + end + op = @(x,mode)linop_fft_r2r( sz, N, bigN, n2, even, x, mode ); + otherwise + error('bad input for cmode: must be "c2c", "r2c", or "r2r"'); +end + +function y = linop_fft_c2c(sz, N, bigN, x, mode) +switch mode, + case 0, y = sz; + case 1, y = fft(x,bigN); % input of size N. Norm is bigN + case 2, + y = bigN*ifft(x); % do NOT use ifft(x,N) because we want to truncate AFTER the ifft, not before + y = y(1:N,:); +end + +function y = linop_fft_r2c(sz, N, bigN, x, mode) +switch mode, + case 0, y = sz; + case 1, + if ~isreal(x), + x = real(x); +% x = real(x+conj(x))/2; % another possibility + end + y = fft(x,bigN); + case 2, + y = bigN*ifft(x,'symmetric'); + y = y(1:N,:); +end + + +function y = linop_fft_r2r(sz, N, bigN, n2, even, x, mode) +switch mode, + case 0, y = sz; + case 1, + error(sizeCheck( x, mode, sz )); + if ~isreal(x), + x = real(x); + end + z = fft(x,bigN); % output is of size bigN + y = real(z); + if even + % y( (n2+2):bigN, : ) = imag( z( 2:n2, : ) ); % convention "A" (not orthogonal) + y( (n2+2):bigN, : ) = -imag( z( n2:-1:2, : ) ); % convention "B" (orthogonal) + y( n2+1, :) = y( n2+1, :)/sqrt(2); + else + % y( (n2+1):bigN, : ) = imag( z( 2:n2, : ) ); % convention "A" + y( (n2+1):bigN, : ) = -imag( z( n2:-1:2, : ) );% convention "B" + end + y = sqrt(2)*y; % the sqrt(2) is so adjoint=inverse + y(1,:) = y(1,:)/sqrt(2); + case 2, + error(sizeCheck( x, mode, sz )); + assert( isreal(x) ); + y = complex(x); % reserve the memory for y + n = bigN; + if even + % y(2:n2, : ) = x(2:n2, : ) + 1i*x((n2+2):n, : ); % convention "A" + y(2:n2, : ) = ( x(2:n2, : ) - 1i*x(n:-1:(n2+2), : ) )/sqrt(2); % convention "B" + + % We can skip this (see the "trick" mentioned below) + % y(n:-1:n2+2, : )= conj( y(2:n2,:) ); + else + % y(2:n2, : ) = x(2:n2, : ) + 1i*x((n2+1):n, : ); % convention "A" + y(2:n2, : ) = ( x(2:n2, : ) - 1i*x(n:-1:(n2+1), : ) )/sqrt(2); % convention "B" + + % We can skip this (see the "trick" mentioned below) + % y(n:-1:n2+1, : )= conj( y(2:n2, :) ); + end + % Note: we are using a trick. We have commented out + % the lines that set y(n2+2:n) (or n2+1:n), because + % the ifft, when using the 'symmetric' option, + % will not look at these entries. + + y = bigN*ifft(y,'symmetric'); + y = y(1:N,:); +end + +function ok = sizeCheck( x, mode, sz ) +szX = size(x); +szY = sz{mode}; +if numel(szX) == numel(sz{1}) && all( szX == szY ) + ok = []; +else + ok = 'Dimensions mismatch in linop_fft; please check your input size. '; + if numel(szX) == 2 + if szX(1) == szY(2) && szX(2) == szY(1) + ok = strcat(ok,' Looks like you have a row vector instead of a column vector'); + end + end +end + +%{ +The conjugate-to-real transformation as a matrix. +(we design this for conjugate-symmetry, but you must make sure + that it still does the same operations when the input + is not conjugate-symmetric) + +n = 10; +even = n/2 == round(n/2) +P = zeros(n); +P(1,1) = 1; % DC component +if even, d = n/2 - 1; +P(d+2,d+2) = 1; % Nyquist component +else, d = (n-1)/2; end + +I = eye(d)/sqrt(2); +J = fliplr(I); + +first=2:d+1; +if even, second=d+3:n; else, second=d+2:n; end + +% Convention "A" +P(first, first) = I; % upper left block +P(second,first) = -1i*I; % lower left block + +P(first, second) = J; % upper right block +P(second,second) = 1i*J; % lower right block + +% or... this makes P' = inv(P). Convention "B" + +P(first, first) = I; % upper left block +P(second,first) = 1i*J; % lower left block. now, J, not I + +P(first, second) = J; % upper right block +P(second,second) = -1i*I; % lower right block. now, I, not J +%} + + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license informatio diff --git a/linop_handles.m b/linop_handles.m new file mode 100644 index 0000000..015adcb --- /dev/null +++ b/linop_handles.m @@ -0,0 +1,89 @@ +function op = linop_handles( sz, Af, At, cmode ) +%LINOP_HANDLES Linear operator from user-supplied function handles. +%OP = LINOP_HANDLES( SZ, AF, AT, CMODE ) +% Constructs a TFOCS-compatible linear operator from separate function +% handles that compute the forward and adjoint operations. The first +% argument, SZ, gives the size of the linear operator; and the forward +% and adjoint handles are AF and AT, respectively. +% +% If the inputs and outputs are simple vectors, then SZ can take the +% standard Matlab form [M,N], where N is the input length and M is the +% output length. If the input or output is a matrix or array, then SZ +% should take the form { S_in, S_out }, where S_in is the size of the +% input and S_out is the size of the output. +% +% If the input or output space of the operator is complex, then the +% CMODE string must be supplied, which describes the forward operation: +% 'R2R': real input and output +% 'R2C': real input, complex output +% 'C2R': complex input, real output +% 'C2C': complex input, complex output +% If CMODE is not supplied, then 'R2R' is assumed. If the operator +% detects a complex input or output when it is not expected, then an +% error results. Therefore, you must make sure that your operators +% return real values when they are expected to do so. + +error(nargchk(3,4,nargin)); +if numel( sz ) ~= 2, + error( 'Size must have two elements.' ); +elseif isnumeric( sz ), + sz = { [sz(2),1], [sz(1),1] }; +elseif ~isa( sz, 'cell' ), + error( 'Invalid operator size specification.' ); +end +if ~isa( Af, 'function_handle' ) || ~isa( At, 'function_handle' ), + error( 'Second and third arguments must be function handles.' ); +end +if nargin < 4 || isempty( cmode ), + cmode = 'R2R'; +elseif ~ischar( cmode ) || size( cmode, 1 ) ~= 1, + error( 'Fourth argument must be a string.' ); +else + cmode = upper( cmode ); +end + +switch cmode, + case 'R2R', op = @(x,mode)linop_handles_r2r( sz, Af, At, x, mode ); + case {'R2C','R2CC'}, op = @(x,mode)linop_handles_r2c( sz, Af, At, x, mode ); + case {'C2R','CC2R'}, op = @(x,mode)linop_handles_c2r( sz, Af, At, x, mode ); + case 'C2C', op = @(x,mode)linop_handles_c2c( sz, Af, At, x, mode ); + otherwise, + error( 'Invalid complex mode: %s', cmode ); +end + +function y = linop_handles_r2r(sz, Af, At, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = realcheck( Af( realcheck( x ) ) ); + case 2, y = realcheck( At( realcheck( x ) ) ); +end + +function y = linop_handles_r2c(sz, Af, At, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = Af( realcheck( x ) ); + case 2, y = realcheck( At( x ) ); +end + +function y = linop_handles_c2r(sz, Af, At, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = realcheck( Af( x ) ); + case 2, y = At( realcheck( x ) ); +end + +function y = linop_handles_c2c(sz, Af, At, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = Af( x ); + case 2, y = At( x ); +end + +function y = realcheck( y ) +if ~isreal( y ), + error( 'Unexpected complex value in linear operation.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/linop_matrix.m b/linop_matrix.m new file mode 100644 index 0000000..53e8f8c --- /dev/null +++ b/linop_matrix.m @@ -0,0 +1,93 @@ +function op = linop_matrix( A, cmode, width ) + +%LINOP_MATRIX Linear operator, assembled from a matrix. +% If A is a real matrix, OP = LINOP_MATRIX( A ) returns a handle to a +% TFOCS linear operator that uses that matrix to implement its size, +% forward, and adjoint operations. +% +% If A is a complex matrix, OP = LINOP_MATRIX( A, MODE ) returns a +% handle to a TFOCS linear operator that uses A to implement its size, +% forward, and linear operator. MODE is a string that tells it how to +% handle its complex inputs/outputs: +% 'R2R': real input, real output (real matrices only) +% 'R2C': real input, complex output +% 'C2R': complex input, real output +% 'C2C': complex input, complex output +% If the operator detects complex input when it is not expecting it, it +% will issue an error. +% +% OP = LINOP_MATRIX( A, MODE, WIDTH ) +% further specifies that A operates on matrices with WIDTH number +% of columns (and output also has WIDTH columns). Default is 1. +% +% Typically, end users do not need to call this routine. You may supply +% matrices directly to TFOCS, and it will call this as needed. + +if ~isnumeric( A ) || ndims( A ) > 2, + error( 'First input must be a matrix.' ); +end +if nargin < 3, width = 1; end +if numel( A ) == 1, + sz = { [], [] }; +else + sz = { [ size(A,2), width ], [ size(A,1), width ] }; +end +if nargin < 2 || isempty( cmode ), + if ~isreal( A ), + error( 'A real/complex mode must be supplied for complex matrices.' ); + end + cmode = 'R2R'; +elseif ~ischar( cmode ) || size( cmode, 1 ) ~= 1, + error( 'Complex mode must be a string.' ); +else + cmode = upper(cmode); +end +switch cmode, + case 'R2R', + if ~isreal( A ), + error( 'An "R2R" operator requires a real matrix.' ); + end + op = @(x,mode)linop_matrix_r2r( sz, A, x, mode ); + case 'R2C', op = @(x,mode)linop_matrix_r2c( sz, A, x, mode ); + case 'C2R', op = @(x,mode)linop_matrix_c2r( sz, A, x, mode ); + case 'C2C', op = @(x,mode)linop_matrix_c2c( sz, A, x, mode ); + otherwise, + error( 'Unexpected complex mode string: %s', op ); +end + +function y = linop_matrix_r2r( sz, A, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = A * realcheck( x ); + case 2, y = A' * realcheck( x ); +end + +function y = linop_matrix_r2c( sz, A, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = A * realcheck( x ); + case 2, y = real( A' * x ); +end + +function y = linop_matrix_c2r( sz, A, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = real( A * x ); + case 2, y = A' * realcheck( x ); +end + +function y = linop_matrix_c2c( sz, A, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = A * x; + case 2, y = A' * x; +end + +function y = realcheck( y ) +if ~isreal( y ), + error( 'Unexpected complex value in linear operation.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/linop_normest.m b/linop_normest.m new file mode 100644 index 0000000..96fe9df --- /dev/null +++ b/linop_normest.m @@ -0,0 +1,79 @@ +function [ nxe, cnt ] = linop_normest( op, cmode, tol, maxiter ) + +%LINOP_NORMEST Estimates the operator norm. +% EST = LINOP_NORMEST( OP ) estimates the induced norm of the operator: +% || OP || = max_{||x||<=1} ||OP(X)|| +% using a simple power method similar to the MATLAB NORMEST functon. +% +% When called with a single argument, LINOP_TEST begins with a real +% initial vector. To test complex operators, use the two-argument +% version LINOP_NORMEST( OP, cmode ), where: +% cmode = 'R2R': real input, real output +% cmode = 'R2C': real input, complex output +% cmode = 'C2R': imag input, imag output +% cmode = 'C2C': complex input, complex output +% +% LINOP_NORMEST( OP, CMODE, TOL, MAXITER ) stops the iteration when the +% relative change in the estimate is less than TOL, or after MAXITER +% iterations, whichever comes first. +% +% [ EST, CNT ] = LINOP_NORMEST( ... returns the estimate and the number +% of iterations taken, respectively. + +if isnumeric( op ), + op = linop_matrix( op, 'C2C' ); +elseif ~isa( op, 'function_handle' ), + error( 'Argument must be a function handle or matrix.' ); +end +x_real = true; +if nargin >= 2 && ~isempty( cmode ), + switch upper( cmode ), + case { 'R2R', 'R2C' }, x_real = true; + case { 'C2R', 'C2C' }, x_real = false; + otherwise, error( 'Invalid cmode: %s', cmode ); + end +end +if nargin < 3 || isempty( tol ), + tol = 1e-8; +end +if nargin < 4 || isempty( maxiter ), + maxiter = 50; +end +sz = op([],0); +if iscell( sz ), + sz = sz{1}; +elseif ~isempty(sz) + sz = [sz(2),1]; +else + % if the input is the identity, it may not have a size associated with it, + % so current behavior is to try an arbitrary size: + sz = [50,1]; +end +cnt = 0; +nxe = 0; +while true, + if nxe == 0, + if x_real, + xx = randn(sz); + else + xx = randn(sz) + 1j*randn(sz); + end + nxe = sqrt( tfocs_normsq( xx ) ); + end + yy = op( xx / max( nxe, realmin ), 1 ); + nye = sqrt( tfocs_normsq( yy ) ); + xx = op( yy / max( nye, realmin ), 2 ); + nxe0 = nxe; + nxe = sqrt( tfocs_normsq( xx ) ); + if abs( nxe - nxe0 ) < tol * max( nxe0, nxe ), + break; + end + cnt = cnt + 1; + if cnt >= maxiter, + break; + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/linop_reshape.m b/linop_reshape.m new file mode 100644 index 0000000..963e483 --- /dev/null +++ b/linop_reshape.m @@ -0,0 +1,34 @@ +function op = linop_reshape( sz_in, sz_out ) + +%LINOP_RESHAPE Linear operator to perform reshaping of matrices. +% op = linop_reshape( sz_in, sz_out ) creates a linear operator that +% uses the matlab 'reshape' function to reshape between sz_in and sz_out. +% Both sz_in and sz_out must be vectors with d elements (for dimension d). +% Assumes the number of elements does not change. +% +% +% Contributed by Graham Coleman, graham.coleman@upf.edu +% See also linop_vec +error( nargchk(nargin,2,2)); +if ~isnumeric(sz_in) || ~isnumeric(sz_out) + error('sz_in and sz_out must be arrays'); +elseif numel(sz_in) ~= numel(sz_out) + error('sz_in and sz_out must have the same number of elements (usually 2)'); +elseif prod( sz_in(:) ) ~= prod( sz_out(:) ) + error('The number of elements cannot change from sz_in to sz_out'); +end +sz_in = sz_in(:).'; +sz_out = sz_out(:).'; + +op = @(x,mode) gkcop_reshape_impl( sz_in, sz_out, x, mode ); + +function y = gkcop_reshape_impl( sz_in, sz_out, x, mode ) +switch mode, + case 0, y = { sz_in, sz_out }; + case 1, y = reshape( x, sz_out ); + case 2, y = reshape( x, sz_in ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/linop_scale.m b/linop_scale.m new file mode 100644 index 0000000..91ab0b3 --- /dev/null +++ b/linop_scale.m @@ -0,0 +1,62 @@ +function op = linop_scale( scale, sz ) + +%LINOP_SCALE Scaling linear operator. +% OP = LINOP_SCALE( scale ) returns a handle to a TFOCS linear operator +% whose forward and adjoint operators are OP(X) = scale * X. +% "scale" must be a real scalar +% +% OP = LINOP_SCALE( scale, size ) gives the scaling operator +% an explicit size. "sz" can be: +% - the empty matrix "[]" (default), which means the size is not +% yet defined; the TFOCS software will attempt to automatically +% determine the size later. +% - a vector [N1,N2] which implies the domain is the set of N1 x N2 matrices +% ( it is also possible to use [N1,N2,N3,....], which +% means the domain is a set of multi-dimensional arrays ) +% - a scalar N, which is equivalent to [N,1], i.e. the set of N x 1 vectors +% Because this is simple scaling, the range will have the same size +% as the domain. +% +% See also linop_compose + +if ~isnumeric( scale ) && numel( scale ) ~= 1, + error( 'Argument must be a scalar.' ); +elseif ~isreal( scale ), + error( 'Argument must be real.' ); +end +if nargin < 2, sz = []; end +if isempty(sz) + szCell = { [], [] }; +elseif isnumeric(sz) + if numel(sz) <= 1, sz = [sz, 1 ]; end + % ensure that "sz" is a row vector + sz = sz(:).'; + szCell = { sz, sz }; +else + error('bad type for size input'); +end + +if scale == 1, + op = @linop_identity; +else + op = @(x,mode)linop_scale_impl( szCell, scale, x, mode ); +end + +function y = linop_scale_impl( sz, scale, y, mode ) +if mode == 0, + y = sz; +else + y = scale * y; +end + +function OK = isSquare( sz ) +OK = true; +for j = 1:length( sz{1} ) + if sz{1}(j) ~= sz{2}(j) + OK = false; break; + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/linop_spot.m b/linop_spot.m new file mode 100644 index 0000000..7f72b58 --- /dev/null +++ b/linop_spot.m @@ -0,0 +1,81 @@ +function op = linop_spot( A, cmode ) + +%LINOP_SPOT Linear operator, assembled from a SPOT operator. +% If A is a real operator, OP = LINOP_SPOT( A ) returns a handle to a +% TFOCS linear operator that uses that object to implement its size, +% forward, and adjoint operations. +% +% If A is a complex operator, OP = LINOP_SPOT( A, MODE ) returns a +% handle to a TFOCS linear operator that uses A to implement its size, +% forward, and linear operator. MODE is a string that tells it how to +% handle its complex inputs/outputs: +% 'R2R': real input, real output (real matrices only) +% 'R2C': real input, complex output +% 'C2R': complex input, real output +% 'C2C': complex input, complex output +% If the operator detects complex input when it is not expecting it, it +% will issue an error. + +if ~isa( A, 'opSpot' ), + error( 'First input must be a SPOT operator.' ); +end +sz = { [ size(A,2), 1 ], [ size(A,1), 1 ] }; +if nargin < 2 || isempty( cmode ), + if ~isreal( A ), + error( 'A real/complex mode must be supplied for complex matrices.' ); + end + cmode = 'R2R'; +elseif ~ischar( cmode ) || size( cmode, 1 ) ~= 1, + error( 'Complex mode must be a string.' ); +else + cmode = upper(cmode); +end +switch cmode, + case 'R2R', + if ~isreal( A ), + error( 'An "R2R" operator requires a real matrix.' ); + end + op = @(x,mode)linop_matrix_r2r( sz, A, x, mode ); + case 'R2C', op = @(x,mode)linop_matrix_r2c( sz, A, x, mode ); + case 'C2R', op = @(x,mode)linop_matrix_c2r( sz, A, x, mode ); + case 'C2C', op = @(x,mode)linop_matrix_c2c( sz, A, x, mode ); + otherwise, + error( 'Unexpected complex mode string: %s', op ); +end + +function y = linop_matrix_r2r( sz, A, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = A * realcheck( x ); + case 2, y = A' * realcheck( x ); +end + +function y = linop_matrix_r2c( sz, A, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = A * realcheck( x ); + case 2, y = real( A' * x ); +end + +function y = linop_matrix_c2r( sz, A, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = real( A * x ); + case 2, y = A' * realcheck( x ); +end + +function y = linop_matrix_c2c( sz, A, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = A * x; + case 2, y = A' * x; +end + +function y = realcheck( y ) +if ~isreal( y ), + error( 'Unexpected complex value in linear operation.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/linop_subsample.m b/linop_subsample.m new file mode 100644 index 0000000..069c6b1 --- /dev/null +++ b/linop_subsample.m @@ -0,0 +1,161 @@ +function op = linop_subsample( sz, omega, SYMMETRIC ) +%LINOP_SUBSAMPLE Subsampling linear operator. +%OP = LINOP_SUBSAMPLE( SZ, OMEGA ) +% vector and matrix subsampling. Depending on SZ and OMEGA, +% this can do row-sampling (e.g. a partial FFT) +% or it can sample specific entries of a matrix (e.g. matrix completion) +% To specify matrix subsampling, set sz = { [n1,n2], [length(omega),1] } +% where the input matrix has size n1 x n2 +% +%OP = LINOP_SUBSAMPLE( OMEGA ) +% works if OMEGA is a sparse matrix, whose nonzero entries +% specify the entries to sample. +% +%OP = LINOP_SUBSAMPLE( ..., SYMMETRIC ) +% If SYMMETRIC is true, then +% forces the domain to be the space of symmetric matrices +% +% Example with FFT: +% FFT = linop_handles([N,N], @(x)fft(x)/sqrt(N), @(x)sqrt(N)*real(ifft(x)) ,'R2C'); +% and to make a row sub-sampled FFT (M <= N): +% rows = randperm(M); +% A = linop_compose( linop_subsample([M,N], rows ), FFT ); + + +% Designed to be used with a fft or dct +% Do we need a separate oversampling version? +% Also, make linop_DCT and linop_FFT ? +% The reason we might want this is that for linop_FFT, +% people will probably use idct as the transpose -- this is not +% correct, due to scaling + +% Help documentation: TBD +% Constructs a TFOCS-compatible linear operator from separate function +% handles that compute the forward and adjoint operations. The first +% argument, SZ, gives the size of the linear operator; and the forward +% and adjoint handles are AF and AT, respectively. +% +% If the inputs and outputs are simple vectors, then SZ can take the +% standard Matlab form [N,M], where N is the input length and M is the +% output length. If the input or output is a matrix or array, then SZ +% should take the form { S_in, S_out }, where S_in is the size of the +% input and S_out is the size of the output. +% + +error(nargchk(1,3,nargin)); +if nargin < 3 + SYMMETRIC = false; +end +if nargin == 1 + omega = sz; + sz = { size(omega), [nnz(omega),1] }; +elseif nargin ==2 && issparse(sz) + SYMMETRIC = omega; + omega = sz; + sz = { size(omega), [nnz(omega),1] }; +end + +if numel( sz ) ~= 2, + error( 'Size must have two elements.' ); +elseif isnumeric( sz ), + sz = { [sz(2),1], [sz(1),1] }; +elseif ~isa( sz, 'cell' ), + error( 'Invalid operator size specification.' ); +end + +dom = sz{1}; +n1 = dom(1); n2 = dom(2); +ran = sz{2}; + +% There are several possibilities. Let x be a point in the domain +%{ + x is a vector. Then omega should be a vector. + We return x(omega), resp. + + x is a matrix. + omega is a vector + This is ambiguous: does the user want x(omega,:) or x(omega)? + omega is a matrix with 2 columns, then assume it is [I,J] + We convert it to linear indices. + omega is a general matrix, more than 2 columns + Not sure what the user means; report an error. + omega is a sparse matrix + We find it's entries, and use those + +%} + +if n2 == 1 + % x is a vector, not a matrix. Simple. +% op = @(x,mode) linop_subsample_vector( sz, omega, x, mode ); + % Allow it to vectorize along rows: + op = @(x,mode) linop_subsample_row( sz, omega, x, mode ); +else + % trickier case. + if issparse(omega) + ind = find(omega); + [I,J] = ind2sub( sz{1}, ind ); +% op = @(x,mode) linop_subsample_matrix( sz, ind, I, J, SYMMETRIC, x, mode ); + elseif isvector(omega) + ind = omega; + [I,J] = ind2sub( sz{1}, ind ); +% op = @(x,mode) linop_subsample_matrix( sz, omega, SYMMETRIC,x, mode ); + elseif size(omega,2) == 2 + ind = sub2ind( sz{1}, omega(:,1), omega(:,2) ); + I = omega(:,1); + J = omega(:,2); +% op = @(x,mode) linop_subsample_matrix( sz, ind, SYMMETRIC,x, mode ); + else + error('omega is not an acceptable size; perhaps you meant it to be sparse?'); + end + % Make sure 'ind' is a column vector + ind = ind(:); + op = @(x,mode) linop_subsample_matrix( sz, ind, I, J, SYMMETRIC, x, mode ); + +end + + +function y = linop_subsample_vector(sz, omega, x, mode ) +switch mode, + case 0, y = sz; + case 1, + if ~isequal( sz{1}, size(x) ), error('input wrong size for vector subsampling'); end + y = x(omega); + case 2, + y = zeros( sz{1} ); + y(omega) = x; +end +function y = linop_subsample_row(sz, omega, x, mode ) +switch mode, + case 0, y = sz; + case 1, + if ~isequal( sz{1}(1), size(x,1) ), error('input wrong size for vector subsampling'); end + y = x(omega,:); + case 2, + y = zeros( sz{1} ); + y = zeros( sz{1}(1), size(x,2) ); + y(omega,:) = x; +end +function y = linop_subsample_matrix(sz, omega, indI, indJ,SYMMETRIC, x, mode ) +switch mode, + case 0, y = sz; + case 1 + S = []; + S.type = '()'; + S.subs = {omega}; + y = subsref(x,S); + case 2, + dom = sz{1}; n1 = dom(1); n2 = dom(2); + y = sparse( indI, indJ, x, n1, n2 ); + if SYMMETRIC + % in future, might update this + % e.g. force omega to only refer to lower part of matrx, + % and then do the update y = y + tril(y,-1)' + y = (y+y')/2; + end +end + + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/linop_test.m b/linop_test.m new file mode 100644 index 0000000..4ab8ba6 --- /dev/null +++ b/linop_test.m @@ -0,0 +1,206 @@ +function varargout = linop_test( op, cmode, maxits ) + +%LINOP_TEST Performs an adjoint test on a linear operator. +% LINOP_TEST( OP ) attempts to verify that a linear operator OP obeys +% the inner product test: = for all x, y. OP must be a +% TFOCS linear operator with hard-coded size information; that is, +% OP([],0) must return valid size info. +% +% When called with a single argument, LINOP_TEST creates real test +% vectors for X and Y. To test complex operators, use the two-argument +% version LINOP_TEST( OP, cmode ), where: +% cmode = 'R2R': real input, real output +% cmode = 'R2C': real input, complex output +% cmode = 'R2CC': real input, conjugate-symmetric complex output +% cmode = 'C2R': complex input, real output +% cmode = 'CC2R': conjugate-symmetric complex input, real output +% cmode = 'C2C': complex input, complex output +% +% The conjugate-symmetric options follow the symmetry conventions of Matlab's FFT. +% +% LINOP_TEST( OP, CMODE, MAXITS ) performs MAXITS iterations of the +% test loop. MAXITS=25 is the default. +% +% myNorm = LINOP_TEST(...) returns an estimate of the myNorm of +% the linear operator. + +error(nargchk(1,3,nargin)); +if isnumeric( op ), + if nargin < 2 || isempty( cmode ) + disp('warning: for matrix inputs, this assumes the cmode is ''C2C'' unless you specify otherwise'); + cmode = 'C2C'; + end + op = linop_matrix( op, cmode ); +end +y_conjSymmetric = false; +x_conjSymmetric = false; +if nargin < 2 || isempty( cmode ), + x_real = true; + y_real = true; +else + switch upper( cmode ), + case 'R2R', x_real = true; y_real = true; + case 'R2C', x_real = true; y_real = false; + case 'R2CC', x_real = true; y_real = false; y_conjSymmetric = true; + case 'C2R', x_real = false; y_real = true; + case 'CC2R', x_real = false; y_real = true; x_conjSymmetric = true; + case 'C2C', x_real = false; y_real = false; + case 'CC2CC', x_real = false; y_real = false; + x_conjSymmetric = true; + y_conjSymmetric = true; % this is probably never going to happen though + otherwise, error( 'Invalid cmode: %s', cmode ); + end +end +if nargin < 3 || isempty(maxits), + maxits = 25; +end +sz = op([],0); +if ~iscell(sz) + sz = { [sz(2),1], [sz(1),1] }; +elseif isempty(sz{1}) + disp('warning: could not detect the size; this often happens when using a scalar input, which represents scaling'); + if isempty( sz{2} ) + disp(' Proceeding under the assumption that the domain is 1D'); + sz{1} = [1,1]; + sz{2} = [1,1]; + else + disp(' Proceeding under the assumption that the domain equals the range'); + sz{1} = sz{2}; + end +elseif isempty(sz{2}) + disp('warning: could not detect the size; this often happens when using a scalar input, which represents scaling'); + disp(' Proceeding under the assumption that the domain equals the range'); + sz{2} = sz{1}; +end +nf = 0; +na = 0; +errs = zeros(1,maxits+1); +nxe = 0; nye = 0; +for k = 1 : maxits, + + % + % The adjoint test + % + + if x_real, + x = randn(sz{1}); + else + x = randn(sz{1})+1j*randn(sz{1}); + if x_conjSymmetric + x = make_conj_symmetrix( x ); + end + end + + if y_real, + y = randn(sz{2}); + else + y = randn(sz{2})+1j*randn(sz{2}); + if y_conjSymmetric + y = make_conj_symmetrix( y ); + end + end + + nx = myNorm(x); + Ax = op(x,1); + nf = max( nf, myNorm(Ax)/nx ); + Ax_y = tfocs_dot( Ax, y ); + + ny = myNorm(y); + Ay = op(y,2); + na = max( na, myNorm(Ay) / ny ); + Ay_x = tfocs_dot( x, Ay ); + + errs(k) = abs(Ax_y-Ay_x)/(nx*ny); + + % + % The myNorm iteration + % + + if nxe == 0, + if x_real, + xx = randn(sz{1}); + else + xx = randn(sz{1}) + 1j*randn(sz{1}); + if x_conjSymmetric + xx = make_conj_symmetrix( xx ); + end + end + nxe = myNorm(xx); + end + yy = op(xx/nxe,1); + nye = max(realmin,myNorm(yy)); + xx = op(yy/nye,2); + nxe = myNorm(xx); + +end + +% +% Use the estimated singular vectors for a final adjoint est +% + +if nxe > 0, + Ax_y = tfocs_dot( op(xx,1), yy ); + Ay_x = tfocs_dot( op(yy,2), xx ); + errs(end) = abs(Ax_y-Ay_x) / (nxe*nye); +end + +% +% Display the output +% + +nmax = max(nye,nxe); +myNorm_err = abs(nye-nxe) / nmax; +peak_err = max(errs) / nmax; +mean_err = mean(errs) / nmax; +rc = { 'complex', 'real', 'complex symmetric' }; +fprintf( 'TFOCS linear operator test:\n' ); +fprintf( ' Input size: [' ); fprintf( ' %d', sz{1} ); fprintf( ' ], %s\n', rc{x_real+1+2*x_conjSymmetric} ); +fprintf( ' Output size: [' ); fprintf( ' %d', sz{2} ); fprintf( ' ], %s\n', rc{y_real+1+2*y_conjSymmetric} ); +fprintf( 'After %d iterations:\n', maxits ); +fprintf( ' myNorm estimates (forward/adjoint/error): %g/%g/%g\n', nye, nxe, myNorm_err ); +fprintf( ' Gains: forward %g, adjoint %g\n', nf, na ); +fprintf( ' Inner product error:\n' ); +fprintf( ' Mean (absolute/relative): %g/%g\n', mean(errs), mean_err ); +fprintf( ' Peak (absolute/relative): %g/%g\n', max(errs), peak_err ); +fprintf( ' (inner product errors should 1e-10 or smaller)\n'); + +good = true; +if myNorm_err/max( nye, nxe ) > 1e-4 + fprintf(' Detected mismatch in forward/adjoint norm estimates. This is potentially a bad sign. Check your implementation\n'); + good = false; +end +if mean_err > 1e-8 + fprintf(' The mean error (relative) is high. This is a bad sign. Check your implementation\n'); + good = false; +end +if good + fprintf(' Allowing for some roundoff error, there are no obvious errors. This is good.\n'); +end + +if nargout > 0 + varargout{1} = mean([nye,nxe]); +end + + +% Improvement as suggested by Graham Coleman +% Allows for 3D arrays. +% This also changes default behavior of 2D arrays +% to now use the Frobenius norm instead of spectral norm. +% This is wise, since it's a much quicker computation. +function y = myNorm(x) +y = norm( x(:) ); + +function y = make_conj_symmetrix( y ) +ny = size( y, 1 ); +y(1,:) = real(y(1,:)); % DC component is 0 +if round(ny/2) == ny/2 % even + y(ny/2+1,:) = real(y(ny/2+1,:)); % Nyquist component is 0 + y( ny:-1:(ny/2+2) ) = conj( y(2:ny/2) ); +else % odd + y( ny:-1:((ny+1)/2+1) ) = conj( y(2:((ny+1)/2)) ); +end + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/linop_vec.m b/linop_vec.m new file mode 100644 index 0000000..6e2a61b --- /dev/null +++ b/linop_vec.m @@ -0,0 +1,40 @@ +function op = linop_vec( sz ) +%LINOP_VEC Matrix to vector reshape operator +%OP = LINOP_VEC( SZ ) +% Constructs a TFOCS-compatible linear operator that reduces a matrix +% variable to a vector version using column-major order. +% This is equivalent to X(:) +% The transpose operator will reshape a vector into a matrix. +% +% The input SZ should of the form [M,N] where [M,N] describe the +% size of the matrix variable. The ouput vector will be of +% length M*N. If SZ is a single entry, then M = N is assumed. +% For advanced usage with multidimensional arrays, +% use the more general linop_reshape function instead. +% +% To do the reverse operation (from vector to matrix), +% use this function together with linop_adjoint. +% +% See also linop_reshape + +if numel(sz) > 2, error('must supply a 2-entry vector'); end +if numel(sz) == 1, sz = [sz(1),sz(1)]; end + +% Switch conventions of the size variable: +sz = { [sz(1),sz(2)], [sz(1)*sz(2),1] }; + +op = @(x,mode)linop_handles_vec( sz, x, mode ); + +function y = linop_handles_vec(sz, x, mode ) +switch mode, + case 0, y = sz; + case 1, y = x(:); + case 2, + MN = sz{1}; M = MN(1); N = MN(2); + y = reshape( x, M, N ); + +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/mexFiles/makeMex.m b/mexFiles/makeMex.m new file mode 100644 index 0000000..166bf00 --- /dev/null +++ b/mexFiles/makeMex.m @@ -0,0 +1,57 @@ +function makeMex() +% Script to install mex files + +% Change directory, so all mex files are in the mexFiles/ subdirectory +here = pwd; +cd( fullfile(tfocs_where,'mexFiles') ); +if exist('OCTAVE_VERSION','builtin') + octave = true; + compileFunction = @compileForOctave; +else + octave = false; + compileFunction = @compileForMatlab; +end + + +% -- Compile all the mex files -- + +% For the ordered L1 norm, used in prox_OL1.m: +compileFunction('proxAdaptiveL1Mex.c'); + + + + +% Change directory back to wherever we started from +cd(here); + +end + + + + +% -- subroutines -- +function compileForMatlab( inputFile ) +mex(inputFile) +end + +function compileForOctave( inputFile ) +if ~isunix, + disp('warning, this will probably fail if not on linux; please edit makeMex.m yourself'); +end +% To compile oct or mex files with octave, you must have the octave function "mkoctfile" +% e.g. (for ubuntu 10.04, using octave 3.2): sudo apt-get install octave3.2-headers +% for other versions of ubuntu and/or octave, try packages like +% octave-pkg-dev or liboctave-dev +system(sprintf('mkoctfile --mex %s', inputFile ) ); + +% rm the .c ending +if length(inputFile) > 2 && inputFile(end-1:end)=='.c' + inputFile = inputFile(1:end-2); +end +system(sprintf('rm %s.o', inputFile ) ); +end + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/mexFiles/proxAdaptiveL1Mex.c b/mexFiles/proxAdaptiveL1Mex.c new file mode 100644 index 0000000..3951eab --- /dev/null +++ b/mexFiles/proxAdaptiveL1Mex.c @@ -0,0 +1,133 @@ +/* This code from http://www-stat.stanford.edu/~candes/OrderedL1/ + * "Statistical Estimation and Testing via the Ordered L1 Norm" + * by M. Bogdan, E. van den Berg, W. Su, and E. J. Candès + * 2013 + * Released under the BSD 3-term license + * See license.txt + * */ + + +#include "mex.h" +#include + +/* Input: Vectors y, lambda, both non-negative and in decreasing order. */ + + +int evaluateProx(double *y, double *lambda, double *x, size_t n); + +/* ----------------------------------------------------------------------- */ +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) +/* ----------------------------------------------------------------------- */ +{ mxArray *vectorY; + mxArray *vectorLambda; + mxArray *vectorX; + mwSize n, i, j, jMax, k; + double *y; + double *lambda; + double *x; + + /* Exit if no parameters are given */ + if (nrhs == 0) + { return ; + } + + /* Check for proper number of arguments */ + if (nrhs != 2) mexErrMsgTxt("Two input arguments expected."); + if (nlhs > 2) mexErrMsgTxt("Too many output arguments."); + + /* Extract the arguments */ + vectorY = (mxArray *)prhs[0]; + vectorLambda = (mxArray *)prhs[1]; + + /* Both vectors should be real, double, n-by-1 */ + n = mxGetNumberOfElements(vectorY); /* For code symmetry */ + if ((!mxIsDouble(vectorY)) || (mxIsComplex(vectorY)) || + (mxGetM(vectorY) != n) || (mxGetN(vectorY) != 1) || + (mxGetNumberOfDimensions(vectorY) != 2)) + { mexErrMsgTxt("First input argument should be a real n-by-1 vector of type double.\n"); + } + + if ((!mxIsDouble(vectorLambda)) || (mxIsComplex(vectorLambda)) || + (mxGetM(vectorLambda) != n) || (mxGetN(vectorLambda) != 1) || + (mxGetNumberOfDimensions(vectorLambda) != 2)) + { mexErrMsgTxt("Second input argument should be a real n-by-1 vector of type double.\n"); + } + + + /* --- DO NOT CHECK DECREASING ORDER AND SIGN PATTERNS --- */ + + + /* Allocate output argument */ + vectorX = mxCreateDoubleMatrix(n,1,mxREAL); + plhs[0] = vectorX; + + /* Get data pointers */ + lambda = mxGetPr(vectorLambda); + y = mxGetPr(vectorY); + x = mxGetPr(vectorX); + + + /* Solve prox function */ + evaluateProx(y,lambda,x,n); + + return ; +} + + + +/* ----------------------------------------------------------------------- */ +int evaluateProx(double *y, double *lambda, double *x, size_t n) +/* ----------------------------------------------------------------------- */ +{ double d; + double *s = NULL; + double *w = NULL; + size_t *idx_i = NULL; + size_t *idx_j = NULL; + size_t i,j,k; + int result = 0; + + /* Allocate memory */ + s = (double *)malloc(sizeof(double) * n); + w = (double *)malloc(sizeof(double) * n); + idx_i = (size_t *)malloc(sizeof(size_t) * n); + idx_j = (size_t *)malloc(sizeof(size_t) * n); + + if ((s != NULL) && (w != NULL) && (idx_i != NULL) && (idx_j != NULL)) + { + k = 0; + for (i = 0; i < n; i++) + { + idx_i[k] = i; + idx_j[k] = i; + s[k] = y[i] - lambda[i]; + w[k] = s[k]; + + while ((k > 0) && (w[k-1] <= w[k])) + { k --; + idx_j[k] = i; + s[k] += s[k+1]; + w[k] = s[k] / (i - idx_i[k] + 1); + } + + k++; + } + + for (j = 0; j < k; j++) + { d = w[j]; if (d < 0) d = 0; + for (i = idx_i[j]; i <= idx_j[j]; i++) + { x[i] = d; + } + } + } + else + { result = -1; + } + + /* Deallocate memory */ + if (s != NULL) free(s); + if (w != NULL) free(w); + if (idx_i != NULL) free(idx_i); + if (idx_j != NULL) free(idx_j); + + return result; +} diff --git a/mexFiles/proxAdaptiveL1Mex.mex b/mexFiles/proxAdaptiveL1Mex.mex new file mode 100755 index 0000000..320b46f Binary files /dev/null and b/mexFiles/proxAdaptiveL1Mex.mex differ diff --git a/mexFiles/proxAdaptiveL1Mex.mexa64 b/mexFiles/proxAdaptiveL1Mex.mexa64 new file mode 100755 index 0000000..4d050e0 Binary files /dev/null and b/mexFiles/proxAdaptiveL1Mex.mexa64 differ diff --git a/private/linop_identity.m b/private/linop_identity.m new file mode 100644 index 0000000..35f6315 --- /dev/null +++ b/private/linop_identity.m @@ -0,0 +1,10 @@ +function y = linop_identity( x, mode ) + +%LINOP_IDENTITY Internal TFOCS routine. +% Implements the identity linear operator. + +y = x; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/private/linop_stack.m b/private/linop_stack.m new file mode 100644 index 0000000..8a5354b --- /dev/null +++ b/private/linop_stack.m @@ -0,0 +1,297 @@ +function [ op, inp_dims, otp_dims ] = linop_stack( linearF, inp_dims, otp_dims, DO_DEBUG ) + +%LINOP_STACK Stacked linear operators. +% OP = LINOP_STACK( linearF ), where linearF is a cell vector or cell +% matrix, returns a function handle for a linear operator that accepts +% TFOCS_TUPLE objects as input or output, as appropriate, and applies +% the various linear operators in block matrix fashion. +% +% If linearF has more than one row, then the output in its forward mode +% or its input in adjoint mode is a TFOCS_TUPLE object. If linearF has +% more than one column, then the output in its adjoint mode or its input +% in forward mode is a TFOCS_TUPLE object. + +if nargin < 4 || isempty(DO_DEBUG), DO_DEBUG = false; end + +if ~isa( linearF, 'cell' ), + error( 'First argument must be a cell array.' ); +end +[ m, n ] = size( linearF ); +if nargin < 2 || isempty( inp_dims ), + inp_dims = cell( 1, n ); +end +if nargin < 3 || isempty( otp_dims ), + otp_dims = cell( 1, m ); +end +rescan = zeros(2,0); +debugPrintf('----- DEBUG INFO: Size of linear matrix (and offsets) ---- \n'); +% debugPrintf('---------------------------------------------------------- \n'); +for j = 1 : n, debugPrintf('-----------------------------------+'); end +debugPrintf('\n'); +% old_inp_d = {}; +for i = 1 : m, + otp_d = otp_dims{i}; + for j = 1 : n, + inp_d = inp_dims{j}; + lF = linearF{i,j}; + sZ = []; + if isempty(lF), + elseif isa( lF, 'function_handle' ), + sZ = lF([],0); + elseif ~isnumeric( lF ), + error( 'Entries should be real matrices or linear operators.' ); + elseif ~isreal(lF), % Why? we now handle A: C --> C + error( 'Matrix entries must be real.' ); + elseif numel(lF) > 1, + sZ = size(lF); + linearF{i,j} = linop_matrix( lF ); % Jan 2012, check this + elseif lF == 0, + linearF{i,j} = []; + else + if lF == 1, +% linearF{i,j} = @(x,mode)x; + linearF{i,j} = @linop_identity; + else + linearF{i,j} = @(x,mode)lF*x; + end + if ~isempty(otp_d), + sZ = { otp_d, otp_d }; + elseif ~isempty(inp_d), + sZ = { inp_d, inp_d }; + else + rescan(:,end+1) = [i;j]; + end + end + if isempty( sZ ), + printSizes( inp_d , otp_d ); % if DO_DEBUG is true, this will print + if j < n, debugPrintf(' |'); end + continue; + elseif isnumeric( sZ ), % This should never be triggered, unless offset is empty + % June 2011: + % If this is the offset term, then we allow for a matrix (rather than vector) + % offset, as long as the size of the linear portion has been specified: + if j == n && j > 1 + sZ_old = linearF{i,j-1}([],0); + if ~isempty( sZ_old) && all( sZ_old{2} == sZ ) + % We may have a matrix + sZ = { [1,1], sZ }; + % So, re-define linearF{i,j} not to be linop_matrix but rather the constant function + else + sZ = { [sZ(2),1], [sZ(1),1] }; + end + else + sZ = { [sZ(2),1], [sZ(1),1] }; + end + end + if isempty(inp_d), + inp_d = sZ{1}; +% elseif ~isequal(inp_d,sZ{1}) && ~isempty( sZ{1} ) % adding Oct 12. Jan 2012, is this right? inp_d was already defined.... +% elseif ~isempty( old_inp_d ) && ~isempty( sZ{1} ) && ~isequal( old_inp_d, sZ{1} ) +% if j > 1 +% for jj = 1:(j-1) +% sZ_old = linearF{i,jj}([],0); +% if isempty( sZ_old ) +% fprintf( 2, ... +% 'TFOCS message: About to throw an error: may be because element (%d,%d) of \n',i,jj); +% fprintf( 2, ... +% ' linear operator matrix does not have an explicit size\n' ); +% end +% end +% end +% error( 'Incompatible dimensions in element (%d,%d) of the linear operator matrix', i, j ); + end + inp_dims{j} = inp_d; +% if ~isempty( inp_d ), old_inp_d = inp_d; end + + if isempty(otp_d), + otp_d = sZ{2}; + elseif ~isequal(otp_d,sZ{2}), + if isequal( fliplr(otp_d), sZ{2} ) + fprintf('\nThe sizes match if you switch some rows/columns. Double-check your offsets are column vectors\n\n'); + end + error( 'Incompatible dimensions in element (%d,%d) of the linear operator matrix', i, j ); + end + + + printSizes( inp_d, otp_d ); % if DO_DEBUG is true, this will print + if j < n, debugPrintf(' |'); end + + end + otp_dims{i} = otp_d; + + debugPrintf('\n'); + +end +debugPrintf('---------------------------------------------------------- \n'); + +% +% In some cases, we cannot resolve the dimensions on the first pass: +% specifically, those entries that represent scalar scaling operations. +% In those cases, we know that the input and output dimensions must be the +% same, but we may not have yet determined either in the first pass. So +% we rescan those entries until all ambiguities are resolved or until no +% further progress is made. +% + +while ~isempty(rescan), + rescan_o = rescan; + rescan = zeros(2,0); + for ij = rescan, + i = ij(1); j = ij(2); + lF = linearF{i,j}; + if isnumeric(lF) && numel(lF) == 1, + if isempty(inp_dims{j}), + if isempty(otp_dims{i}), + rescan(:,end+1) = [i;j]; + continue; + else + inp_dims{j} = otp_dims{i}; + end + elseif isempty(otp_dims{i}), + otp_dims{i} = inp_dims{j}; + elseif ~isequal( inp_dims{i}, otp_dims{j} ), + error( 'Incompatible dimensions in element (%d,%d) of the linear operator matrix', i, j ); + end + if DO_DEBUG + fprintf('Affine term (%d,%d) has size:', i, j ); + printSizes( inp_dims{j}, otp_dims{i} ); + end + end + end + % Prevent infinite loops + if numel(rescan) == numel(rescan_o), + break; + end +end +debugPrintf('---------------------------------------------------------- \n'); + +if m == 1 && n == 1, + op = linearF{1,1}; + inp_dims = inp_dims{1}; + otp_dims = otp_dims{1}; + if isempty(op), + op = @linop_identity; + end +elseif m == 1, + otp_dims = otp_dims{1}; + op = @(x,mode)linop_stack_row( linearF, n, { inp_dims, otp_dims }, x, mode ); +elseif n == 1, + inp_dims = inp_dims{1}; + op = @(x,mode)linop_stack_col( linearF, m, { inp_dims, otp_dims }, x, mode ); +else + op = @(x,mode)linop_stack_mat( linearF, [m,n], { inp_dims, otp_dims }, x, mode ); +end + + +% ------- Internal subfunctions ------------ +% These functions can see the workspace variables of the main function +function debugPrintf( varargin ) +if DO_DEBUG + fprintf( varargin{:} ); +end +end + +function printSizes( inp_d, otp_d ) +if DO_DEBUG + % Print size of domain + if isempty( inp_d ) + fprintf(' ( ? )'); + else + if length( inp_d ) == 1, inp_d = [inp_d, 1 ]; end + fprintf(' ('); + for kk = 1:length(inp_d)-1, fprintf('%4d x ', inp_d(kk) ); end + fprintf('%4d )', inp_d(end) ); + end + % Print size of range + fprintf(' --> '); + if isempty( otp_d ) + fprintf('( ? )'); + else + if length( otp_d ) == 1, otp_d = [otp_d, 1 ]; end + fprintf('('); + for kk = 1:length(otp_d)-1, fprintf('%4d x ', otp_d(kk) ); end + fprintf('%4d )', otp_d(end) ); + end +end +end + + +end % end of main program + +% ------- External subfunctions ------------ +function y = linop_stack_row( linearF, N, dims, x, mode ) +switch mode, + case 0, + y = dims; + case 1, + y = 0; + x = cell( x ); + for j = 1 : N, + lF = linearF{j}; + if ~isempty(lF), y = y + lF(x{j},1); end + end + case 2, + y = cell(1,N); + for j = 1 : N, + lF = linearF{j}; + if ~isempty(lF), y{j} = lF(x,2); else y{j} = 0*x; end + end + y = tfocs_tuple( y ); +end +end + +function y = linop_stack_col( linearF, N, dims, x, mode ) +switch mode, + case 0, + y = dims; + case 1, + y = cell(1,N); + for j = 1 : N, + lF = linearF{j}; + if ~isempty(lF), y{j} = lF(x,1); else y{j} = 0*x; end + end + y = tfocs_tuple( y ); + case 2, + y = 0; + x = cell( x ); + for j = 1 : N, + lF = linearF{j}; + if ~isempty(lF), y = y + lF(x{j},2); end + end +end +end + +function y = linop_stack_mat( linearF, sZ, dims, x, mode ) +switch mode, + case 0, + y = dims; + case 1, + x = cell( x ); + y = cell( 1, sZ(1) ); + for i = 1 : sZ(1), + ans = 0; + for j = 1 : sZ(2), + lF = linearF{i,j}; + if ~isempty(lF), ans = ans + lF(x{j},1); end + end + y{i} = ans; + end + y = tfocs_tuple( y ); + case 2, + x = cell( x ); + y = cell( 1, sZ(2) ); + for j = 1 : sZ(2), + ans = 0; + for i = 1 : sZ(1), + lF = linearF{i,j}; + if ~isempty(lF), ans = ans + lF(x{i},2); end + end + y{j} = ans; + end + y = tfocs_tuple( y ); +end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/private/print_cell_size.m b/private/print_cell_size.m new file mode 100644 index 0000000..20a0f89 --- /dev/null +++ b/private/print_cell_size.m @@ -0,0 +1,40 @@ +function print_cell_size( c , fid, offsetPossible ) +% c should be a cell array + +if nargin < 2 || isempty(fid), fid = 1; end +if nargin < 3 || isempty(offsetPossible), offsetPossible = false; end + +if ~iscell(c) + fprintf(fid,'\tcomponent 1: '); +% d = size(c); + d = c; + for k = 1:(length(d)-1) + fprintf(fid,'%4d x ',d(k) ); + end + fprintf(fid,'%4d\n', d(k+1) ); + fprintf(fid,' (but input to printCellSize should have been a cell array)\n'); + return; +else + for j = 1:length(c) + if j == length(c) && offsetPossible && all(size( c{j} ) == [1,2] ) ... + && all( c{j} == [1,1] ) + fprintf(fid,'\tcomponent %2d is fixed (i.e. an offset)\n', j ); + else + fprintf(fid,'\tcomponent %2d: ', j ); + if isempty( c{j} ) + fprintf(fid,'size not yet determined\n'); + else + d = c{j}; + if length(d) < 2, d = [d,1]; end % this case shouldn't arise... + for k = 1:(length(d)-1) + fprintf(fid,'%4d x ',d(k) ); + end + fprintf(fid,'%4d\n', d(k+1) ); % bug, Feb 29 2012: change d(k) to d(k+1) + end + end + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. \ No newline at end of file diff --git a/private/prox_stack.m b/private/prox_stack.m new file mode 100644 index 0000000..08bd4dd --- /dev/null +++ b/private/prox_stack.m @@ -0,0 +1,64 @@ +function op = prox_stack( varargin ) + +% OP = PROJ_STACK( P1, P2, P3, ..., PN ) +% "Stacks" N proximity functions P1, P2, P3, ..., PN together, to create +% a single proximity function that operates on an N-tuple. Returns a +% function handle ready to be used in + +args = varargin; +while isa( args, 'cell' ) && numel( args ) == 1, + args = args{1}; +end +if isempty( args ), + op = @proj_Rn; +elseif isa( args, 'function_handle' ), + op = args; +elseif ~isa( args, 'cell' ), + error( 'Expected one or more projector function handles.' ); +else + op = @(varargin)proj_stack_impl( args, varargin{:} ); +end + +function [ v, x ] = proj_stack_impl( proj, y, t ) + +np = numel(proj); +no = nargout > 1; +ni = nargin > 2; +y = cell( y ); +if no, + v = zeros( 1, np ); + x = cell( 1, np ); +else + v = 0; +end +if ni && numel(t) == 1, + t = t * ones(1,np); +end +switch 2 * no + ni, +case 0, % 1 input, 1 output + for k = 1 : np, + v = v + proj{k}( y{k} ); + end +case 1, % 2 inputs, 1 output + for k = 1 : np, + v = v + proj{k}( y{k}, t(k) ); + end +case 2, % 1 input, 2 outputs + for k = 1 : np, + [ v(k), x{k} ] = proj{k}( y{k} ); + end +case 3, % 2 inputs, 2 outputs + v = zeros( 1, np ); + x = cell( 1, np ); + for k = 1 : np, + [ v(k), x{k} ] = proj{k}( y{k}, t(k) ); + end +end +if no, + v = sum( v ); + x = tfocs_tuple( x ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/private/size_ambig.m b/private/size_ambig.m new file mode 100644 index 0000000..e4ca632 --- /dev/null +++ b/private/size_ambig.m @@ -0,0 +1,26 @@ +function ans = size_ambig( sz ) + +switch class( sz ), + case 'double', + ans = isempty( sz ); + case 'cell', + if isa( sz{1}, 'function_handle' ), + ans = false; % it's not ambiguous as long as it's a valid function... + elseif isempty( sz ), + ans = true; + else + ans = false; + for k = 1 : numel( sz ), + if size_ambig( sz{k} ), + ans = true; + break; + end + end + end + otherwise, + ans = true; +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/private/size_compat.m b/private/size_compat.m new file mode 100644 index 0000000..53b65da --- /dev/null +++ b/private/size_compat.m @@ -0,0 +1,57 @@ +function [ a, sX ] = size_compat( sX, sY ) +a = true; +switch class( sX ), + case 'double', + if isempty( sX ) || all( sX == 0 ), + sX = sY; + elseif isempty( sY ) || all( sY == 0 ), + elseif ~isequal( sX, sY ), + + % Feb 29, 2012. Special case: + % One represents the size a x b x c, where c = 1 + % The other is a x b (since Matlab often automatically squeezes + % 3D arrays to 2D if the 3rd dimension is a singletone) +% if (length(sX) >= 3 && length(sX) == length(sY)+1 && sX(end)==1) || ... +% (length(sY) >= 3 && length(sY) == length(sX)+1 && sY(end)==1) + % do nothing + % March 2012, a better fix (also due to Graham Coleman) + if min( length(sX),length(sY) ) >= 2 + truncA = cutTrailingOnes(sX); + truncB = cutTrailingOnes(sY); + a = isequal( truncA, truncB ); + else + a = false; + end + end + case 'cell', + if ~isa( sY, 'cell' ) || numel( sX ) ~= numel( sY ) || isa( sX{1}, 'function_handle' ) && ~isequal( sX, sY ), + a = false; + elseif isa( sX{1}, 'function_handle' ), + a = isequal( sX, sY ); + else + for k = 1 : numel( sX ), + [ ta, sX{k} ] = size_compat( sX{k}, sY{k} ); + a = a && ta; + end + end + otherwise, + a = isequal( sX, sY ); +end +if ~a, + sX = []; +end + +function y = cutTrailingOnes( x ) +%cuts the final ones of a vector, and columnize + +lastNotOne = find( x(:)~=1, 1, 'last' ); + +%do not cut before 2nd position +lastNotOne = max( 2, lastNotOne ); + +y = x(1:lastNotOne); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/private/smooth_stack.m b/private/smooth_stack.m new file mode 100644 index 0000000..9c28f08 --- /dev/null +++ b/private/smooth_stack.m @@ -0,0 +1,43 @@ +function op = smooth_stack( varargin ) + +% OP = SMOOTH_STACK( S1, S2, S3, ..., SN ) +% "Stacks" N smooth functions S1, S2, S3, ..., SN together to create +% a single smooth function that operates on an N-tuple. Returns a +% function handle ready to be used in TFOCS. + +args = varargin; +while isa( args, 'cell' ) && numel( args ) == 1, + args = args{1}; +end +if isempty( args ), + op = @smooth_zero; +elseif isa( args, 'function_handle' ), + op = args; +elseif ~isa( args, 'cell' ), + error( 'Expected one or more smooth function handles.' ); +else + op = @(varargin)smooth_stack_impl( args, varargin{:} ); +end + +function [ f, g ] = smooth_stack_impl( smooth, x ) + +np = numel(smooth); +x = cell( x ); +if nargout > 1, + f = zeros( 1, np ); + g = cell( 1, np ); + for k = 1 : np, + [ f(k), g{k} ] = smooth{k}( x{k} ); + end + f = sum( f ); + g = tfocs_tuple( g ); +else + f = 0; + for k = 1 : np, + f = f + smooth{k}( x{k} ); + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/private/solver_apply.m b/private/solver_apply.m new file mode 100644 index 0000000..6fe5254 --- /dev/null +++ b/private/solver_apply.m @@ -0,0 +1,13 @@ +function varargout = solver_apply( ndxs, func, varargin ) +global tfocs_count___ + +%SOLVER_APPLY Internal TFOCS routine. +% A wrapper function used to facilitate the counting of function calls, and +% to standardize the number of output arguments for the algorithms. + +[ varargout{1:max(nargout,1)} ] = func( varargin{:} ); +tfocs_count___(ndxs) = tfocs_count___(ndxs) + 1; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/private/tfocs_backtrack.m b/private/tfocs_backtrack.m new file mode 100644 index 0000000..f232288 --- /dev/null +++ b/private/tfocs_backtrack.m @@ -0,0 +1,37 @@ +% TFOCS_BACKTRACK +% Backtracking helper script. + +% Quick exit for no backtracking +if beta >= 1, break; end % SRB changing == to >= + +% Quick exit if no progress made +xy = x - y; +xy_sq = tfocs_normsq( xy ); +if xy_sq == 0, localL = Inf; break; end + +% Compute Lipschitz estimate +if backtrack_simple, + if isinf( f_x ), + f_x = apply_smooth( A_x ); + end + q_x = f_y + tfocs_dot( xy, g_y ) + 0.5 * L * xy_sq; + localL = L + 2 * max( f_x - q_x, 0 ) / xy_sq; + backtrack_simple = abs( f_y - f_x ) >= backtrack_tol * max( abs( f_x ), abs( f_y ) ); +else + if isempty( g_Ax ), + [ f_x, g_Ax ] = apply_smooth( A_x ); + end + localL = 2 * tfocs_dot( A_x - A_y, g_Ax - g_Ay ) / xy_sq; +end + +% Exit if Lipschitz criterion satisfied, or if we hit Lexact +backtrack_steps = backtrack_steps + 1; +if localL <= L || L >= Lexact, break; end +if ~isinf( localL ), + L = min( Lexact, localL ); +elseif isinf( localL ), localL = L; end +L = min( Lexact, max( localL, L / beta ) ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/private/tfocs_cleanup.m b/private/tfocs_cleanup.m new file mode 100644 index 0000000..3f35689 --- /dev/null +++ b/private/tfocs_cleanup.m @@ -0,0 +1,62 @@ +%SOLVER_CLEANUP TFOCS helper script +% Performs the final set of operations for our templated solvers: performs +% final calculations, prints final outputs, cleans up unused data. + +if saddle, + if isempty(g_Ax), + [ f_x, g_Ax ] = apply_smooth(A_x); + end + out.dual = get_dual( g_Ax ); + % Oct 13: make it compatible: + if isa( out.dual, 'tfocs_tuple') + out.dual = cell( out.dual ); + end +elseif isinf(f_x), + f_x = apply_smooth(A_x); +end +if isinf( C_x ), + C_x = apply_projector( x ); +end +if fid && printEvery, + fprintf( fid, 'Finished: %s\n', status ); +end +out.niter = n_iter; +out.status = status; +d.niter = 'Number of iterations'; +if saveHist, + out.f(n_iter) = maxmin * ( f_x + C_x ); + out.f(n_iter+1:end) = []; + out.normGrad(n_iter+1:end) = []; + out.stepsize(n_iter+1:end) = []; + out.theta(n_iter+1:end,:) = []; + if countOps, + out.counts(n_iter+1:end,:) = []; + end + if ~isempty(errFcn), + out.err(n_iter+1:end,:) = []; + end + d.f = 'Objective function history'; + d.normDecr = 'Decrement norm'; + d.stepsize = 'Stepsize'; + d.theta = 'Acceleration parameter history'; + if countOps, + d.counts = strvcat(... + 'k x 4 arry, with columns [F,G,A,P] where',... + 'F: Number of function evaluations of the smooth function',... + 'G: Number of gradient evaluations of the smooth function',... + 'A: Number of calls to the linear operator and its transpose',... + 'N: Number of calls to the nonsmooth function (w/o projection)',... + 'P: Number of calls to the projection operator' ); + end + if ~isempty(errFcn) + d.err = 'Error, determined by evaluating the user-supplied error function'; + end +end +out.description = d; +if countOps, + clear global tfocs_count___ +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/private/tfocs_initialize.m b/private/tfocs_initialize.m new file mode 100644 index 0000000..4b15eb9 --- /dev/null +++ b/private/tfocs_initialize.m @@ -0,0 +1,598 @@ +%SOLVER_INITIALIZE TFOCS helper script +% Performs the initializations common to all of the first-order solvers + +% Process the options structure and string/value pairs. First, we replace the +% default values with any values specified by the user in the 'opts' structure. +% We ignore any fields in the 'opts' structure that do not match ours, so that +% users can re-use the opts structure for other purposes. + +odef = struct( ... + 'maxIts', Inf ,... + 'maxCounts', Inf ,... + 'countOps', false ,... % adjusted if maxCounts < Inf + 'saveHist', true ,... + 'adjoint', false ,... + 'saddle', false ,... + 'tol', 1e-8 ,... + 'errFcn', {{}} ,... + 'stopFcn', {{}} ,... + 'printEvery', 100 ,... + 'maxmin', 1 ,... + 'beta', 0.5 ,... + 'alpha', 0.9 ,... % See below for special CG stuff + 'L0', 1 ,... % See below + 'Lexact', Inf ,... + 'mu', 0 ,... + 'fid', 1 ,... + 'stopCrit', 1 ,... % the TFOCS paper used stopCrit = 2 + 'alg', 'AT' ,... + 'restart', Inf ,... + 'printStopCrit', false,... + 'cntr_reset', 50 ,... % how often to explicitly recompute A*x and A*y (set to Inf if you want ) + 'cg_restart', Inf ,... % for CG only + 'cg_type', 'pr' ,... % for CG only + 'debug', false .... + ); + +% Calling the solver with a no arguments returns the default options structure +if nargin < 1 || ( nargin ==1 && isstruct( smoothF ) ) + opts = odef; + % remove the "CG" options for now, since they are undocumented + opts = rmfield(opts,'cg_restart'); + opts = rmfield(opts,'cg_type'); + % if any default options are passed in (e.g. by tfocs_SCD), add those now: + if ( nargin ==1 && isstruct( smoothF ) ) + for f = fieldnames( smoothF )' + opts.( f{1} ) = smoothF.( f{1} ); + end + end + + out = []; + x = opts; + if nargout == 0, + disp('====== TFOCS default options ======='); +% disp('Format is fieldname: { [default] ''Usage type. +% Description''}'); + + % add some helpful description + desc = []; + desc.alg = 'Algorithm, e.g. GRA for gradient descent. Values: AT, GRA, LLM, N07, N83, TS'; + desc.maxIts = 'Basic. Maximum number of allowed iterations'; + desc.saveHist = 'Basic. Record history of all iterations'; + desc.restart = 'Basic. Restart parameter. Make negative for "no-regress" feature'; + desc.tol = 'Basic. Tolerance for stopping criteria'; + desc.L0 = 'Basic. Initial estimate of Lipschitz constant'; + desc.Lexact = 'Basic. Known bound of Lipschitz constant'; + desc.printEvery = 'Basic. How often to print info to the terminal; set to 0 or Inf to suppress output'; + desc.maxmin = 'Basic. +1 for convex minimization, -1 for concave maximization'; + + desc.errFcn = 'Medium. User-specified error function. See user guide'; + desc.beta = 'Medium. Backtracking parameter, in (0,1). No line search if >= 1'; + desc.alpha = 'Medium. Line search increase parameter, in (0,1)'; + + + desc.maxCounts = 'Advanced. Vector that fine-tunes various types of iteration limits; same form as countOps'; + desc.countOps = 'Advanced. Record number of linear multiplies, etc.; form [fcn, grad, linear, nonsmth, proj]'; + desc.mu = 'Advanced. Strong convexity parameter.'; + desc.fid = 'Advanced. File id, e.g. via fopen. All output is sent to this file'; + desc.stopFcn = 'Advanced. User-supplied stopping criteria. See user guide'; + desc.stopCrit = 'Advanced. Controls which stopping criteria to use; 1,2, 3 or 4.'; + desc.printStopCrit = 'Advanced. Controls whether to display the value used in the stopping criteria'; + desc.cntr_reset = 'Advanced. Controls how often to reset some numerical computatios to avoid roundoff'; + desc.debug = 'Advanced. Turns on more useful error messages'; + + desc.adjoint = 'Internal.'; + desc.saddle = 'Internal. Used by TFOCS_SCD'; + + + + disp( opts ); + disp('====== Description of TFOCS options ======='); + disp( desc ); + end + return % from this script only; the calling function does not return +end + +% [smoothF, affineF, projectorF, x0, opts ] = deal(varargin{:}); +% Check for incorrect types +F_types = {'function_handle','cell','double','single'}; +assert( ismember(class(smoothF),F_types),'smoothF is of wrong type' ); +assert( ismember(class(affineF),F_types),'affineF is of wrong type' ); +assert( ismember(class(projectorF),F_types),'projectorF 3 is of wrong type' ); +if nargin >= 4 + x0_types = {'cell','double','single','tfocs_tuple'}; + assert( ismember(class(x0),x0_types),'x0 is of wrong type' ); +end +if nargin >= 5 && ~isempty(opts) + assert( ismember(class(opts),{'struct'}),'opts is of wrong type' ); +end + +% Some parameters defaults depend on whether the user supplies other +% options. These will be updated later +L0_default = odef.L0; +odef.L0 = Inf; + +alpha_default = odef.alpha; % typically 0.9 +alpha_default_CG = 1; +odef.alpha = Inf; + + + +% Process the options structure, merging it with the default options +% (merge "opts" into "odef") +def_fields = fieldnames(odef)'; % default options +if nargin > 4 && ~isempty(opts), + use_fields = zeros(size(def_fields)); + opt_fields = fieldnames(opts)'; % user-supplied options + for k = opt_fields, + k = k{1}; + ndx = find(strcmpi(def_fields,k)); + if ~isempty(ndx) + if ~isempty(opts.(k)) && ~use_fields(ndx), + odef.(def_fields{ndx}) = opts.(k); + end + use_fields(ndx) = use_fields(ndx) + 1; + else + % Warn if the field is not found in our options structure + warnState = warning('query','backtrace'); + warning off backtrace + warning('TFOCS:OptionsStructure',' Found extraneous field "%s" in options structure', k ); + warning(warnState); + end + end + % Warn if fields appear twice due to capitalization; e.g., maxIts/maxits + if any(use_fields>1), + warnState = warning('query','backtrace'); + warning off backtrace + warning('TFOCS:OptionsStructure',' Some fieldnames of the options structure are identical up to capitalization: unpredictable behavior'); + warning(warnState); + end +end + +% Remove unnecessary options +if ~strcmpi( odef.alg, 'cg' ) + odef = rmfield( odef, 'cg_restart' ); + odef = rmfield( odef, 'cg_type' ); + % update our list of fieldnames + def_fields = fieldnames(odef)'; +end + +% If opts.alpha wasn't supplied, use a default value: +if isinf(odef.alpha) + if strcmpi( odef.alg, 'cg' ) + odef.alpha = alpha_default_CG; + else + odef.alpha = alpha_default; + end +end + +% If opts.printEvery is Inf, set it to 0 +if isinf(odef.printEvery) + odef.printEvery = 0; +end + +% The default value of L0 is set to Lexact, if it is supplied +if isinf(odef.L0), + if isinf(odef.Lexact), + if odef.beta >= 1, + error( 'For a fixed step size, L0 or Lexact must be supplied.' ); + end + odef.L0 = L0_default; + else + odef.L0 = odef.Lexact; + end +end +% If maxCounts is set, set countOps to true +if any(odef.maxCounts 1, + error( 'Multidimensional arrays are not permitted.' ); + end + if numel(affineF) == 1, + identity_linop = affineF == 1; + affineF = { linop_scale( affineF ) }; + else + identity_linop = false; + affineF = { linop_matrix( affineF ) }; + end +elseif isa( affineF, 'function_handle' ), + affineF = { affineF }; +elseif ~isa( affineF, 'cell' ), + error( 'Invalid affine operator specification.' ); +end + +% If adjoint mode is specified, temporarily transpose the affineF cell +% array so that the rows and columns match the number of smooth functions +% and projectors, respectively. Then verify size compatibility. +if adjoint, affineF = affineF'; end +[ m_aff, n_aff ] = size( affineF ); +% if all( m_aff ~= n_smooth + [0,1] ) || n_proj && all( n_aff ~= n_proj + [0,1] ), +% error( 'The affine operation matrix has incompatible dimensions.' ); +% May 16, 2011: making error messages more informative +if all( m_aff ~= n_smooth + [0,1] ) + if fid + fprintf(fid,'Detected error: inputs are of wrong size\n'); + fprintf(fid,' Found %d smooth functions, so expect that many primal variables\n',... + n_smooth ); + fprintf(fid,' So the affine operator should have %d (or %d if there is an offset) entries\n',... + n_smooth, n_smooth+1 ); + fprintf(fid,' but instead found %d affine operators\n', m_aff ); + fprintf(fid,' Perhaps you need the transpose of the affine operator?\n'); + end + error( 'The affine operation matrix has incompatible dimensions.' ); +elseif n_proj && all( n_aff ~= n_proj + [0,1] ), + if fid + fprintf(fid,'Detected error: inputs are of wrong size\n'); + fprintf(fid,' Found %d nonsmooth functions, so expect %d or %d sets of affine operators\n',... + n_proj, n_proj, n_proj + 1 ); + fprintf(fid,' but instead found %d affine operators\n', n_aff ); + end + error( 'The affine operation matrix has incompatible dimensions.' ); +elseif n_proj == 0, + n_proj = max( 1, m_aff - 1 ); +end +inp_dims = cell( 1, n_proj ); +otp_dims = cell( 1, n_smooth ); + +% If an additional affine portion of the objective has been specified, +% create an additional smooth function to contain it. +if m_aff == n_smooth + 1, + offset = true; % note: saddle_ndxs is 1:n_smooth + otp_dims{end+1} = [1,1]; + smoothF{end+1} = smooth_linear( maxmin ); + n_smooth = n_smooth + 1; + for k = 1 : n_proj, + offX = affineF{end,k}; + if isempty(offX), + affineF{end,k} = 0; + elseif isa(offX,'function_handle'), + if adjoint, pos = 'row'; else pos = 'column'; end + error( 'The elements in the last %s must be constants.', pos ); + elseif isnumeric(offX) && numel(offX) == 1 && offX == 0, + affineF{end,k} = 0; + elseif nnz(offX), + affineF{end,k} = linop_dot( affineF{end,k}, adjoint ); + % add a case if ~nnz(offX)... Jan 2012 + elseif ~nnz(offX) + affineF{end,k} = 0; + end + end +else + offset = false; +end + +% If an affine offset has been specified, integrate those offsets into +% each smooth function, then remove that portion of the array. +if n_aff == n_proj + 1, + for k = 1 : n_smooth, + offX = affineF{k,end}; + if isempty(offX), + continue; + elseif isa(offX,'function_handle'), + if adjoint, pos = 'column'; else pos = 'row'; end + error( 'The elements in the last %s must be constant matrices.', pos ); + elseif isnumeric(offX) && numel(offX) == 1 && offX == 0, + continue; + else + otp_dims{k} = size(offX); + if nnz(offX), + smoothF{k} = @(x)smoothF{k}( x + offX ); + end + end + end + n_aff = n_aff - 1; + affineF(:,end) = []; +end + + +% -- Todo: +% If opts.debug = true, then before calling linop_stack, +% we should print a message showing the sizes of everything +% (this is useful when the linear portion is entered as "1" or "[]" +% and we have automatically determined the size ). + +% Transpose back, if necessary; then check dimensions +if adjoint, + affineF = affineF'; + [ linearF, otp_dims, inp_dims ] = linop_stack( affineF, otp_dims, inp_dims, debug ); + linearF = linop_adjoint( linearF ); +else + [ linearF, inp_dims, otp_dims ] = linop_stack( affineF, [], [], debug ); +end +identity_linop = isequal( linearF, @linop_identity ); % doesn't always catch identities +square_linop = identity_linop || isequal( inp_dims, otp_dims ); +adj_arr = [0,2,1]; +if countOps, + apply_linear = @(x,mode)solver_apply( 3, linearF, x, mode ); +else + apply_linear = linearF; +end + +% +% Smooth function, pass 2: integrate scale, counting +% + +smoothF = smooth_stack( smoothF ); +if maxmin < 0, + smoothF = tfunc_scale( smoothF, -1 ); +end +if countOps, + apply_smooth = @(x)solver_apply( 1:(1+(nargout>1)), smoothF, x ); +else + apply_smooth = smoothF; +end + +% +% Projector, pass 2: supply default, stack it up, etc. +% + +if isempty( projectorF ), + n_proj = 0; + projectorF = proj_Rn; +else + projectorF = prox_stack( projectorF ); +end +if countOps, + apply_projector = @(varargin)solver_apply( 4:(4+(nargout>1)), projectorF, varargin{:} ); +else + apply_projector = projectorF; +end + +% +% Initialize the op counts +% + +if countOps, + global tfocs_count___ + tfocs_count___ = [0,0,0,0,0]; + maxCounts = maxCounts(:)'; +end + +% +% Construct the common initial values +% + +L = L0; +theta = Inf; +f_v_old = Inf; +x = []; A_x = []; f_x = Inf; C_x = Inf; g_x = []; g_Ax = []; +restart_iter = 0; +warning_lipschitz = 0; +backtrack_simple = true; +backtrack_tol = 1e-10; +backtrack_steps = 0; + +% +% Try to determine the size of the input, and construct the initial point. +% + +% Attempt 1: From x0 itself +zero_x0 = true; +if ~isempty( x0 ), + if isa( x0, 'tfocs_tuple' ) + x0 = cell(x0); + end + if isa( x0, 'cell' ), + n_x0 = numel( x0 ); + if n_x0 == 1, + x0 = x0{1}; + else + x0 = tfocs_tuple( x0 ); + end + else + n_x0 = 1; + end + if n_proj && n_proj ~= n_x0, + error( 'Size conflict detected between the projector and x0.' ); + end + zero_x0 = ~nnz(x0); +% Attempt 2: From the linear operator dimensions +elseif ~size_ambig( inp_dims ), + x0 = tfocs_zeros( inp_dims ); +elseif ~size_ambig( otp_dims ), + A_x = tfocs_zeros( otp_dims ); + x0 = apply_linear( A_x, 2 ); +end +if isempty( x0 ), + error( 'Could not determine the dimensions of the problem. Please supply an explicit value for x0.' ); +end +x = x0; +if isinf( C_x ), + C_x = apply_projector( x ); + if isinf( C_x ), + zero_x0 = false; + [ C_x, x ] = apply_projector( x, 1 ); + end +end +if isempty( A_x ), + if identity_linop || zero_x0 && square_linop, + A_x = x; + elseif ~zero_x0 || size_ambig( otp_dims ), % Jan 2012: todo: give size_ambig the 'offset' information + A_x = apply_linear( x, 1 ); % celldisp( size(A_x) ) + else + A_x = tfocs_zeros( otp_dims ); + end +end + +% New, Jan 2012 +if debug + if ~adjoint, str1 = 'primal'; str2 = 'dual'; + else, str1 = 'dual'; str2 = 'primal'; + end + fprintf(fid,'------- DEBUG INFO -----------\n'); + offsetPossible = offset && ~adjoint; + fprintf(fid,'Size of %s variable, via method 1\n', str1); + print_cell_size( size(x0), fid, offsetPossible); + fprintf(fid,'Size of %s variable, via method 2\n', str1); + print_cell_size( inp_dims, fid, offsetPossible); + + offsetPossible = offset && adjoint; + fprintf(fid,'Size of %s variable, via method 1\n', str2); + print_cell_size( size(A_x), fid, offsetPossible); + fprintf(fid,'Size of %s variable, via method 2\n', str2); + print_cell_size( otp_dims, fid, offsetPossible); + fprintf(fid,'------------------------------\n'); +end +% Added Dec 2012, check for bad sizes that are not integers +if ~isequal( otp_dims, tfocs_round(otp_dims) ) + error('Output dimensions must be integer valued'); +end +if ~isequal( inp_dims, tfocs_round(inp_dims) ) + error('Input dimensions must be integer valued'); +end + +% Final size check +[ isOK1, inp_dims ] = size_compat( size(x0), inp_dims ); +[ isOK2, otp_dims ] = size_compat( size(A_x), otp_dims ); +if ~isOK1 || ~isOK2, + if debug + if ~isOK1 + fprintf(fid,'Debug message: size of %s variables did not line up\n',str1); + end + if ~isOK2 + fprintf(fid,'Debug message: size of %s variables did not line up\n',str2); + end + end + error( 'Could not determine the dimensions of the problem. Please supply an explicit value for x0.' ); +end +[ f_x, g_Ax ] = apply_smooth( A_x ); +if isinf( f_x ), + error( 'The initial point lies outside of the domain of the smooth function.' ); +end +% Adding Feb 2011: +if isa(g_Ax,'tfocs_tuple') + get_dual = @(g_Ax) get( g_Ax, saddle_ndxs ); +else + get_dual = @(g_Ax) g_Ax; +end + +% Theta advancement function +% if mu > 0 && Lexact > mu +if mu > 0 && ~isinf(Lexact) && Lexact > mu, % fixed Dec 2011 + theta_scale = sqrt(mu / Lexact); + theta_scale = ( 1 - theta_scale ) / ( 1 + theta_scale ); + advance_theta = @(theta_old,L,L_old) min(1,theta_old*theta_scale); +else + advance_theta = @(theta_old,L,L_old) 2/(1+sqrt(1+4*(L/L_old)/theta_old.^2)); +end + +% Preallocate the arrays for the output structure +out.alg = alg; +out.algorithm = algorithm; +if ~isempty(errFcn) && ~iscell(errFcn), + errFcn = { errFcn }; +end +if ~isempty(stopFcn) && ~iscell(stopFcn), + stopFcn = { stopFcn }; +end +errs = zeros(1,length(errFcn)); +if nargout == 1, + saveHist = false; +end +if saveHist, + [ out.f, out.normGrad, out.stepsize, out.theta ] = deal( zeros(0,1) ); + if countOps, + out.counts = zeros(0,length(tfocs_count___)); + end + if ~isempty(errFcn), + out.err = zeros(0,length(errs)); + end +end +if saddle, + out.dual = []; +end +n_iter = 0; +status = ''; + +% Initialize the iterate values +y = x; z = x; +A_y = A_x; A_z = A_x; +C_y = C_x; C_z = C_x; +f_y = f_x; f_z = f_x; +g_y = g_x; g_z = g_x; +g_Ay = g_Ax; g_Az = g_Ax; +norm_x = sqrt( tfocs_normsq( x ) ); + +% for recomputing linear operators +cntr_Ay = 0; +cntr_Ax = 0; + +% Special setup for constant step sizes +if beta >= 1, + beta = 1; + alpha = 1; +end + +% Print the opening text +if fid && printEvery, + fprintf(fid,'%s\n',algorithm); + fprintf(fid,'Iter Objective |dx|/|x| step'); + if countOps, fprintf( fid, ' F G A N P' ); end + if ~isempty(errFcn) + nBlanks = max( [0, length(errFcn)*9 - 9] ); + fprintf( fid, ' errors%s',repmat(' ',nBlanks,1) ); + end + if printStopCrit, fprintf( fid, ' stopping criteria' ); end + fprintf( fid, '\n' ); + fprintf(fid,'----+----------------------------------' ); + if countOps, fprintf( fid, '+-------------------------------' ); end +% if ~isempty(errFcn), fprintf( fid, '+-------------------' ); end + if ~isempty(errFcn) + fprintf( fid, '+%s', repmat('-',1+length(errFcn)*9,1) ); + end + + + if printStopCrit, fprintf( fid, '+%s', repmat('-',19,1) ); end + + + fprintf(fid,'\n'); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/private/tfocs_iterate.m b/private/tfocs_iterate.m new file mode 100644 index 0000000..0ac3bab --- /dev/null +++ b/private/tfocs_iterate.m @@ -0,0 +1,284 @@ +%SOLVER_ITERATE TFOCS helper script +% Performs the iterate processing common to all of the first-order solvers +% +% Major inputs: x, x_old, xy , A_x, A_y, g_Ax, g_Ay +% (does not need g_x or g_y really) + +% Test for positive stopping criteria +n_iter = n_iter + 1; +norm_x = sqrt( tfocs_normsq( x ) ); +norm_dx = sqrt( tfocs_normsq( x - x_old ) ); +% We also handle legacy stopping criteria used in the paper: +if stopCrit == 2 && beta >= 1 + % xy_sq not already computed + xy = x - y; + xy_sq = tfocs_normsq( xy ); +end +if isnan( f_y ), + status = 'NaN found -- aborting'; +elseif (stopCrit == 1) && norm_dx == 0 + if n_iter > 1 + status = 'Step size tolerance reached (||dx||=0)'; + end +elseif (stopCrit == 1) && norm_dx < tol * max( norm_x, 1 ), + status = 'Step size tolerance reached'; +elseif (stopCrit == 2) && L*sqrt(xy_sq) < tol, + status = 'Step size tolerance reached'; +elseif n_iter == maxIts, + status = 'Iteration limit reached'; +elseif countOps && any( tfocs_count___ >= maxCounts ), + status = 'Function/operator count limit reached'; +elseif backtrack_steps > 0 && xy_sq == 0, + status = 'Unexpectedly small stepsize'; +end + +% for stopCrit 3, we need the new and old dual points +if stopCrit == 3 || stopCrit == 4 + if ~saddle, error('stopCrit = {3,4} requires a saddle point problem'); end + if exist('cur_dual','var') + old_dual = cur_dual; + else + old_dual = []; + end +end + +% +% For stopping criteria or algorithm control, we assume that the user +% needs the objective function value, but does not wish to do any more +% computation than necessary. So we will use the function value for y +% instead of x if that is cheaper to obtain. So here we determine what +% the additional computational costs will be, and choose the path that +% minimizes them, favoring x in the case of a tie. +% + +% if isempty(status) && ( ~isempty(stopFcn) || restart < 0 || stopCrit == 3 || stopCrit ==4 ), +if (isempty(status) || ~isempty(findstr(status,'limit')) ) ... + && ( ~isempty(stopFcn) || restart < 0 || stopCrit == 3 || stopCrit ==4 ), + comp_x = [ isinf(f_x), saddle*~isempty(stopFcn)*isempty(g_Ax), isinf(C_x) ]; + comp_y = [ isinf(f_y), saddle*~isempty(stopFcn)*isempty(g_Ay), isinf(C_y) ]; + if sum(comp_x) <= sum(comp_y), + if comp_x(2), [f_x,g_Ax] = apply_smooth(A_x); + elseif comp_x(1), f_x = apply_smooth(A_x); end + if comp_x(3), C_x = apply_projector(x); end + cur_pri = x; cur_dual = g_Ax; + f_v = maxmin*(f_x+C_x); + else + if comp_y(2), [f_y,g_Ay] = apply_smooth(A_y); + elseif comp_y(1), f_y = apply_smooth(A_y); end + if comp_y(3), C_y = apply_projector(y); end + cur_pri = y; cur_dual = g_Ay; + f_v = maxmin*(f_y+C_y); + end + for err_j = 1 : numel(stopFcn), + if saddle, + stop = stopFcn{err_j}(f_v,cur_pri, get_dual(cur_dual) ); + else + stop = stopFcn{err_j}(f_v,cur_pri); + end + if stop + status = sprintf('Reached user''s supplied stopping criteria no. %d',err_j); + end + end +end + +% Now we can apply stopCrit 3 if it has been requested: +if (stopCrit == 3 || stopCrit == 4 ) + if ~isempty( old_dual ) && ~isempty( cur_dual ) + d_dual = sqrt(tfocs_normsq( old_dual - cur_dual )); + if isnumeric( cur_dual ), + norm_cur = tfocs_normsq( cur_dual ); + norm_old = tfocs_normsq( old_dual ); + else + norm_cur = 0; + norm_old = 0; + for j = 1:max( [1, numel(cur_dual)-1] ) + norm_cur = norm_cur + tfocs_normsq( cur_dual{j} ); + norm_old = norm_old + tfocs_normsq( old_dual{j} ); + end + end + norm_cur = sqrt(norm_cur); + norm_old = sqrt(norm_old); + else + d_dual = Inf; + norm_cur = 0; + norm_old = 0; + end + + % Note: it is common for the duals to be stuck at zero for quite a few + % iterations at the beginning. In stopCrit = 4 mode, we will not + % terminate if both the current and old dual are zero. + + if stopCrit == 4 + % for "4", we look at relative change, not absolute change + if norm_cur > 10*eps && norm_old > 10*eps + d_dual = d_dual / norm_cur; + else + % The dual vectors are zero, so do not terminate. + % This is equivalent to defining 0/0 = Inf; + d_dual = Inf; + end + end + if d_dual < tol + status = 'Step size tolerance reached'; + end +end + + + +% +% Data collection. Since this is used only for algorithm analysis and not +% for algorithm control, we are free to make additional computations here +% without adding them to our total algorithm cost. We prefer to track the +% x sequence in this case, since it is what we will ultimately choose as +% the solution at the end of the algorithm. +% + +will_print = fid && printEvery && ( ~isempty( status ) || ~mod( n_iter, printEvery ) ); +if saveHist || will_print, + f_x_save = f_x; + g_Ax_save = g_Ax; + if ~isempty(errFcn) && saddle, + if isempty(g_Ax), + [ f_x, g_Ax ] = smoothF( A_x ); + end + out.dual = get_dual( g_Ax ); + end + if isinf(f_x), + f_x = smoothF( A_x ); + end + if isinf( C_x ), + C_x = projectorF( x ); + end + f_w = maxmin * ( f_x + C_x ); + if ~isempty(errFcn) && iscell(errFcn) + for err_j = 1 : numel(errFcn), + if saddle, + errs(err_j) = errFcn{err_j}(f_w,x,out.dual); + else + errs(err_j) = errFcn{err_j}(f_w,x); + end + end + end + f_x = f_x_save; + g_Ax = g_Ax_save; +end + +% Register a warning if the step size suggests a Lipschitz violation +if isempty(status) && ( beta < 1 && backtrack_simple && localL > Lexact ), + warning_lipschitz = true; +end + +% Print status +if will_print, + if warning_lipschitz, + warning_lipschitz = false; + bchar = 'L'; + elseif backtrack_simple, + bchar = ' '; + else + bchar = '*'; + end + fprintf( fid, '%-4d| %+12.5e %8.2e %8.2e%c', ... + n_iter, f_w, norm_dx / max( norm_x, 1 ), 1 / L, bchar ); + if countOps, + fprintf( fid, '|' ); + fprintf( fid, ' %5d', tfocs_count___ ); + end + if ~isempty(errFcn), + if countOps, + fprintf( fid, ' ' ); + end + fprintf( fid, '|' ); + fprintf( fid, ' %8.2e', errs ); + end + + + if printStopCrit + % Display the number used to determine stopping + + switch stopCrit + case 1 + if exist('norm_dx','var') && exist('norm_x','var') + stopResid = norm_dx/max( norm_x,1); + else + stopResid = Inf; + end + case 2 + stopResid = L*sqrt(xy_sq); + case {3,4} + if exist('d_dual','var') + stopResid = d_dual; + end + case Inf + % do nothing + stopResid = 0; + otherwise + error('Bad stopCrit value'); + end + + if ~isempty(errFcn) || countOps + fprintf( fid, ' ' ); + end + fprintf( fid, '|' ); + fprintf( fid, ' %8.2e', stopResid ); + end + + fprintf( fid, '\n'); +end + +% Save history, extending arrays if needed +if saveHist, + if length(out.f) < n_iter && isempty(status), + csize = min(maxIts,length(out.f)+1000); + out.f(end+1:csize,1) = 0; + out.theta(end+1:csize,1) = 0; + out.stepsize(end+1:csize,1) = 0; + out.normGrad(end+1:csize,1) = 0; + if countOps, + out.counts(end+1:csize,:) = 0; + end + if ~isempty(errs), + out.err(end+1:csize,:) = 0; + end + end + out.f(n_iter) = f_w; + out.theta(n_iter) = theta; + out.stepsize(n_iter) = 1 / L; + out.normGrad(n_iter) = norm_dx; + if countOps, + out.counts(n_iter,:) = tfocs_count___; + end + if ~isempty(errFcn), + out.err(n_iter,:) = errs; + end +end + +% Exit if positive stopping criteria met +if ~isempty( status ), + break; +end + +% Restart acceleration if necessary +backtrack_steps = 0; +% "No regress" feature: test was (maxmin*f_v > f_v_old) +% This worked for minimization, but not for maximization! Dec 15 2010. +% Fixed it. +% two changes: (1) test is now ( maxmin*f_v > maxmin*f_v_old) +% (2) to reset f_v_old, set to maxmin*Inf, not just +Inf +if n_iter - restart_iter == abs(round(restart)) || (restart < 0 && maxmin*f_v > maxmin*f_v_old), + restart_iter = n_iter; + backtrack_simple = true; + theta = Inf; + y = x; A_y = A_x; f_y = f_x; g_Ay = g_Ax; g_y = g_x; C_y = C_x; + z = x; A_z = A_x; f_z = f_x; g_Az = g_Ax; g_z = g_x; C_z = C_x; + f_v_old = maxmin*Inf; % important! +% continue; +elseif restart < 0, + f_v_old = f_v; +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + + diff --git a/private/tfocs_prox.m b/private/tfocs_prox.m new file mode 100644 index 0000000..b17d0e1 --- /dev/null +++ b/private/tfocs_prox.m @@ -0,0 +1,113 @@ +function op = tfocs_prox( f, prox_f, VECTOR_SCALAR ) +% OP = TFOCS_PROX( F, PROX_F ) +% combines F and PROX_F into the appropriate TFOCS-compatible object. +% +% F is any function (with known proximity operator), +% and PROX_F is its proximity operator, defined as: +% +% PROX_F( Y, t ) = argmin_X F(X) + 1/(2*t)*|| X - Y ||^2 +% +% To use this, please see the file PROX_L1 as an example +% +% The basic layout of a file like PROX_L1 is as follows: +% ( for the function F(X) = q*||X||_1 ) +% +% function op = prox_l1(q) +% op = tfocs_prox( @f, @prox_f ) +% +% function v = f(x) +% ... this function calculates the function f ... +% end +% function v = prox_f(y,t) +% ... this function calculates the prox-function to f ... +% end +% end +% +% Note: in the above template, the "end" statements are very important. +% +% OP = TFOCS_PROX( F, PROX_F, 'vector') +% will signal the routine that it is OK to allow "vector" stepsizes +% (this corresponds to solving +% PROX_F( Y, T ) = argmin_X F(X) + 1/2( X-Y )'*diag(1./T)*( X-Y ) +% where T is a vector of the same size as X ). +% ... = TFOCS_PROX( F, PROX_F, 'scalar' ) +% will throw an error if a vector stepsize is attempted. This is the default. +% +% Also, users may wish to test their smooth function +% with the script TEST_NONSMOOTH +% +% See also prox_l1, test_nonsmooth + +if nargin < 3 || isempty(VECTOR_SCALAR) + VECTOR_SCALAR = 'scalar'; +end + +if strcmpi(VECTOR_SCALAR,'scalar') +% op = @fcn_impl; % comment this out and use octave-compatible version below: +% op = @(x,t)fcn_impl(f,prox_f,x,t); % this doesn't work: requires 2 inputs always + op = @(varargin)fcn_impl(f,prox_f,varargin{:}); +elseif strcmpi(VECTOR_SCALAR,'vector') +% op = @fcn_impl_vector; % comment this out and use octave-compatible version below: +% op = @(x,t)fcn_impl_vector(f,prox_f,x,t); % this doesn't work: requires 2 inputs always + op = @(varargin)fcn_impl_vector(f,prox_f,varargin{:}); +else + error('bad option for VECTOR_SCALAR parameter'); +end + +function [ v, x ] = fcn_impl(f,prox_f,x, t ) + if nargin < 3, + error( 'Not enough arguments.' ); + end + if nargin == 4, + if numel(t) ~= 1, error('The stepsize must be a scalar'); end + x = prox_f(x,t); + elseif nargout == 2, + error( 'This function is not differentiable.' ); + end + v = f(x); +end + +function [ v, x ] = fcn_impl_vector(f,prox_f,x, t ) + if nargin < 3, + error( 'Not enough arguments.' ); + end + if nargin == 4, + x = prox_f(x,t); + elseif nargout == 2, + error( 'This function is not differentiable.' ); + end + v = f(x); +end + +% - Octave incompatible version: - +% function [ v, x ] = fcn_impl(x, t ) +% if nargin < 1, +% error( 'Not enough arguments.' ); +% end +% if nargin == 2, +% if numel(t) ~= 1, error('The stepsize must be a scalar'); end +% x = prox_f(x,t); +% elseif nargout == 2, +% error( 'This function is not differentiable.' ); +% end +% v = f(x); +% end +% +% function [ v, x ] = fcn_impl_vector(x, t ) +% if nargin < 1, +% error( 'Not enough arguments.' ); +% end +% if nargin == 2, +% x = prox_f(x,t); +% elseif nargout == 2, +% error( 'This function is not differentiable.' ); +% end +% v = f(x); +% end + + +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. \ No newline at end of file diff --git a/private/tfocs_round.m b/private/tfocs_round.m new file mode 100644 index 0000000..1617426 --- /dev/null +++ b/private/tfocs_round.m @@ -0,0 +1,10 @@ +function y = tfocs_round(x) +if iscell( x ), + y = cellfun( @tfocs_round, x, 'UniformOutput', false ); +else + y = round( x ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/private/tfocs_smooth.m b/private/tfocs_smooth.m new file mode 100644 index 0000000..1b17a47 --- /dev/null +++ b/private/tfocs_smooth.m @@ -0,0 +1,45 @@ +function op = tfocs_smooth( fcn ) +% OP = TFOCS_SMOOTH(FCN) +% is a wrapper designed to facilitate users writing their own +% smooth functions. +% +% To use this, please see the file SMOOTH_HUBER as an example +% +% The basic layout of a file like SMOOTH_HUBER is as follows: +% +% function op = smooth_huber(mu) +% op = tfocs_smooth( @huber_impl ) +% +% function [f,g] = huber_impl(x) +% ... this function calculates the function, f, +% and the gradient, g, of x ... +% end +% end +% +% Note: in the above template, the "end" statements are very important. +% +% +% Also, users may wish to test their smooth function +% with the script TEST_SMOOTH +% +% See also smooth_huber, test_smooth, private/tfocs_smooth, smooth_handles + + +op = @fcn_impl; + +function [ v, g ] = fcn_impl(x, t ) + if nargin == 2, + error( 'Proximity minimization not supported by this function.' ); + end + if nargout == 2 + [v,g] = fcn(x); + else + v = fcn(x); + end +end + +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. \ No newline at end of file diff --git a/private/tfocs_zeros.m b/private/tfocs_zeros.m new file mode 100644 index 0000000..c8bf8f0 --- /dev/null +++ b/private/tfocs_zeros.m @@ -0,0 +1,54 @@ +function z = tfocs_zeros( y ) +switch class( y ), + case 'double', + % SRB: this was the old code. This function was intended + % to be called as tfocs_zeros( size(y) ) + % But, for packSVD, I'd prefer to call it as tfocs_zeros(y) + % so that we can look at the type of y and pass it off + % accordingly (see 'packSVD' section below). + % This would break old code. But, it appears that all + % calls of this use the { [n1,n2], [m1,m2] } form of sz + % (as opposed to the [m1,n1] convention), so + % I don't think this will break anything. + + % Old code: + % z = zeros( y ); + + % New code: + % Try to catch cases where we thing y = [m,n] is a size: +% if numel(y) == 2 && isint(y(1)) && isint(y(2)) + % March 2011, fixing this to work with 3D arrays + % If you want to use 4D arrays, you must modify this in a similar + % fasion; we don't do that before hand since it makes it more likely + % that the wrong case is picked. + if (numel(y) == 2 || numel(y) == 3) && all(isint(y)) + z = zeros( y ); + else + z = zeros( size(y) ); + end + + case 'cell', + if isa( y{1}, 'function_handle' ), + z = y{1}( y{2:end} ); + else + for k = 1 : numel(y), + y{k} = tfocs_zeros( y{k} ); + end + z = tfocs_tuple( y ); + end + case 'packSVD' + %z = packSVD>tfocs_zeros(y); + z = packSVD_zeros(y); + otherwise + error('TFOCS_ZEROS: cannot handle this type of object'); +end + +% do NOT use "isinteger" because that tests for the integer data type. +function h = isint(y) +h = abs( round(y) - y ) < 10*abs(y)*eps; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + + diff --git a/proj_0.m b/proj_0.m new file mode 100644 index 0000000..2cb3ec5 --- /dev/null +++ b/proj_0.m @@ -0,0 +1,63 @@ +function op = proj_0(offset) + +%PROJ_0 Projection onto the set {0} +% OP = PROJ_0 returns an implementation of the indicator +% function for the set including only zero. +% +% OP = PROJ_0( c ) returns an implementation of the +% indicator function of the set {c} +% If c is a scalar, this is interpreted as c*1 +% where "1" is the all ones object of the appropriate size. +% +% See also prox_0.m, proj_Rn.m and smooth_constant.m, +% which are the Fenchel conjugates of this function. + +if nargin == 0 + op = @proj_0_impl; +else + op = @(varargin) proj_0_impl_q( offset, varargin{:} ); +end + +function [ v, x ] = proj_0_impl( x, t ) +v = 0; +switch nargin, + case 1, + if nargout == 2, + error( 'This function is not differentiable.' ); + elseif any( x(:) ), + v = Inf; + end + case 2, + % "t" variable has no effect + x = 0*x; + otherwise, + error( 'Not enough arguments.' ); +end +function [ v, x ] = proj_0_impl_q( c, x, t ) +v = 0; +switch nargin, + case 2, + if nargout == 2, + error( 'This function is not differentiable.' ); + end + if isscalar(c) + if any( x(:) - c ) + v = Inf; + end + elseif any( x(:) - c(:) ), + v = Inf; + end + case 3, + % "t" variable has no effect + if isscalar(c) && ~isscalar(x) + x = c*ones( size(x) ); + else + x = c; + end + + otherwise, + error( 'Not enough arguments.' ); +end +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_Rn.m b/proj_Rn.m new file mode 100644 index 0000000..99e4948 --- /dev/null +++ b/proj_Rn.m @@ -0,0 +1,14 @@ +function op = proj_Rn + +%PROJ_RN "Projection" onto the entire space. +% OP = PROX_RN returns an implementation of the set Rn. Use +% this function to specify a model with no nonsmooth component. It is +% identical to both PROX_0 and SMOOTH_CONSTANT( 0 ). +% The projection onto Rn is just the identity. +% Dual: proj_0.m + +op = smooth_constant( 0 ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_Rplus.m b/proj_Rplus.m new file mode 100644 index 0000000..1cb078e --- /dev/null +++ b/proj_Rplus.m @@ -0,0 +1,62 @@ +function op = proj_Rplus( scale ) + +%PROJ_RPLUS Projection onto the nonnegative orthant. +% OP = PROJ_RPLUS returns an implementation of the indicator +% function for the nonnegative orthant. +% OP2 = PROJ_RPLUS( scale ) returns the indicator function +% of the scaled nonnegative orthant: that is, +% OP2( x ) = OP( scale * x ). +% Because the nonnegative orthant is a cone, scaling has no +% effect if scale > 0. If scale < 0, the result is an +% indicator function for the nonpositive orthant, which is +% also the conjugate of the original. If scale == 0, then +% the result is the zero function. +% Dual: proj_Rplus(-1) +% +% See also proj_psd.m, the matrix-analog of this function + +if nargin == 0, + op = @proj_Rplus_impl; +elseif ~isa( scale, 'double' ) || numel( scale ) ~= 1 || ~isreal( scale ), + error( 'The argument must be a real scalar.' ); +elseif scale > 0, + op = @proj_Rplus_impl; +elseif scale < 0, + op = @proj_Rminus_impl; +else + op = proj_Rn; +end + +function [ v, x ] = proj_Rplus_impl( x, t ) +v = 0; +switch nargin, + case 1, + if nargout == 2, + error( 'This function is not differentiable.' ); + elseif any( x(:) < 0 ), + v = Inf; + end + case 2, + x = max( x, 0 ); + otherwise, + error( 'Not enough arguments.' ); +end + +function [ v, x ] = proj_Rminus_impl( x, t ) +v = 0; +switch nargin, + case 1, + if nargout == 2, + error( 'This function is not differentiable.' ); + elseif any( x(:) > 0 ), + v = Inf; + end + case 2, + x = min( x, 0 ); + otherwise, + error( 'Not enough arguments.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_affine.m b/proj_affine.m new file mode 100644 index 0000000..be268ae --- /dev/null +++ b/proj_affine.m @@ -0,0 +1,85 @@ +function op = proj_affine( A, b, R ) +%PROJ_AFFINE(A, b) Projection onto the affine set +% { x : A*x == b } +% +% Warning! For large dimensions, this may be costly. +% In this case, use proj_0 and explicitly seprate +% out the affine term (this is the motivation behind +% TFOCS). +% +% For efficiency, this function does a single costly +% Cholesky decomposition when instatiated; subsequent +% calls are cheap. +%PROJ_AFFINE(A, b, R) +% uses the user-provided R for the Cholesky factorization +% of A*A', i.e., R'*R = A*A' and R is upper triangular. +% +% If A is a tight frame, e.g., A*A' = alpha*I for some +% scalar alpha>0, then the projection is efficient, but it is +% not automatically recognized. The user should +% provide R = sqrt(alpha)*eye(m) (where A is m x n) +% and then the code will be efficient. +% +% N.B. If the system of equations is overdetermined, +% then the set is just a single point. +% +% See also proj_boxAffine and proj_singleAffine + +error(nargchk(2,3,nargin)); +if nargin < 3 + if size(A,1) > 1e4 + warning('proj_affine:largeSize','Cholesky decomposition might take a large for such large matrices'); + end + [R,p] = chol(A*A'); +else + p = 0; + % Assuming that if user provides R, then it is not over-determiend +end +if p > 0 + % A*A' is rank deficient, i.e., system is over-determined + % So, there is just a unique point + warning('proj_affine:overdetermined','The system is over-determined so the set is a single point'); + xStar = A\b; + op = @(varargin) proj_point( xStar, varargin{:} ); +else + op = @(varargin) proj_affine_internal( A, R, b, varargin{:} ); +end + +function [v,x] = proj_point( xStar, y, t ) +v = 0; +switch nargin + case 2 + if nargout == 2 + error('This function is not differentiable.'); + end + % Check if we are feasible + if norm( y - xStar ) > 1e-13 + v = Inf; + end + case 3 + % The projection is simple: + x = xStar; + otherwise + error( 'Wrong number of arguments.' ); +end + +function [v,x] = proj_affine_internal( A, R, b, y, t ) +v = 0; +switch nargin + case 4 + if nargout == 2 + error('This function is not differentiable.'); + end + % Check if we are feasible + if norm( A*y - b ) > 1e-13 + v = Inf; + end + case 5 + x = y + A'*( R\( R'\( b - A*y )) ); + otherwise + error( 'Wrong number of arguments.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_box.m b/proj_box.m new file mode 100644 index 0000000..9435766 --- /dev/null +++ b/proj_box.m @@ -0,0 +1,46 @@ +function op = proj_box(l,u) +%PROJ_BOX Projection onto box constraints. +%PROJ_BOX(l,u) Projection onto the box { l <= x <= u } +% If l or u is the empty matrix [], then the constraint is not +% enforced (e.g. PROJ_BOX([],1) is the set { x <= 1 }, +% and PROJ_BOX(0) is the set { 0 <= x } ) +% The parameters l and u may also be vectors or matrixes +% of the same size as x. +% +% OP = PROJ_BOX returns an implementation of this projection. +% +% Dual function: prox_boxDual.m +% See also prox_boxDual + +% warning: doesn't check to ensure l <= u + +error(nargchk(1,2,nargin)); +if nargin < 2, u = []; end +%op = @proj_Rplus_impl; +% bugfix, March 2011: +op = @(varargin)proj_Rplus_impl(l,u,varargin{:}); + +function [ v, x ] = proj_Rplus_impl( l, u, x, t ) +v = 0; +switch nargin, + case 3, + if nargout == 2, + error( 'This function is not differentiable.' ); + end + if any( x < l ) || any( x > u ) + v = Inf; + end + case 4, + if ~isempty(l) + x = max( x, l ); + end + if ~isempty(u) + x = min( x, u ); + end + otherwise, + error( 'Wrong number of arguments.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_boxAffine.m b/proj_boxAffine.m new file mode 100644 index 0000000..aeb8acc --- /dev/null +++ b/proj_boxAffine.m @@ -0,0 +1,111 @@ +function op = proj_boxAffine( a, l, u, alpha ) +%PROJ_BOXAFFINE(a,l,u,alpha) Projection onto the box { l <= x <= u } intersected +% with the constraint a'*x == alpha +% If l or u is the empty matrix [], then the constraint is not +% enforced. "a" must be included (otherwise, use proj_box.m). +% The parameters l and u may also be vectors or matrixes +% of the same size as x. +% The offset "alpha" is optional (default is 0) +% +% OP = PROJ_BOXAFFINE returns an implementation of this projection. +% +% See also proj_box, proj_affine, proj_singleAffine, prox_boxDual +error(nargchk(3,4,nargin)); +if isempty(l), l = -Inf(size(a)); end +if isempty(u), u = Inf(size(a)); end +if nargin < 4 || isempty(alpha), alpha = 0; end +% % Not the most efficient, but least amount of coding work: +% if isequal(size(l),[1,1]) && numel(a) > 1 +% l = l*ones(size(a)); +% end +% if isequal(size(u),[1,1]) && numel(a) > 1 +% u = u*ones(size(a)); +% end +op = @(varargin) proj_box_affine(a,l,u, alpha, varargin{:} ); + + +function [v,x] = proj_box_affine( a, l, u, alpha, y, t ) +v = 0; +switch nargin + case 5 + if nargout == 2 + error('This function is not differentiable.'); + end + % Check if we are feasible + if any( y < l ) || any( y > u ) || abs( tfocs_dot(y,a) ) > 1e-13 + v = Inf; + end + case 6 + scalarL = ( numel(l) == 1 ); + scalarU = ( numel(u) == 1 ); + + n = length(y); + if numel(y) > n + % It should work, but no 100% guarantees +% warning('TFOC:proj_boxAffine','Not extensively tested for matrix inputs; use at your own risk!'); + n = numel(y); + end + projBox = @(x) max( l, min( u, x ) ); + % Turning points for constraints = l (l for lower) + T1 = (y-l)./a; + % Turning points for constraints = u (u for upper) + T2 = (y-u)./a; + T = sort(union(T1(:),T2(:))); + lwrBound = 1; + uprBound = 2*n; + for i = 1:ceil(log2(2*n)) + indx = round( (lwrBound+uprBound)/2 ); + beta = T(indx); + + % Our trial solution + x = projBox( y - beta*a ); + % Refine beta on the support (ignore constraints): see if it satisifies constraints + S = find( x > l & x < u ); + S1 = x==l; + S2 = x==u; + + % Given the fixed points, we subtract these off, and resolve the + % plain affine projection problem on the active set + % The affine projection solun is always x = y - betaEst*a + % for some betaEst. + +% betaEst = (a(S)'*y(S) + a(S2)'*u(S2) + a(S1)'*l(S1) )/(a(S)'*a(S)); + % Or, so that we can allow u and l to be scalars, + if scalarL, al = sum(a(S1))*l; else al = a(S1)'*l(S1); end + if scalarU, au = sum(a(S2))*u; else au = a(S2)'*u(S2); end + betaEst = (a(S)'*y(S) + au + al - alpha )/(a(S)'*a(S)); + + + % Check if bestEst is in the admissible range + % e.g. if betaEst > beta, is it less than T(indx+1)? and vice-versa + if betaEst > beta + if indx == 2*n || betaEst < T(indx+1) + break; + else + lwrBound = indx + 1; % we need to increase beta + end + else + if indx == 1 || betaEst > T(indx-1) + break; + else + uprBound = indx - 1; % we need to decrease beta + end + end + end + x = projBox( y - betaEst*a ); + S = find( x > l & x < u ); + S1 = x==l; + S2 = x==u; +% betaEst = (a(S)'*y(S) + a(S2)'*u(S2) + a(S1)'*l(S1) )/(a(S)'*a(S)); + % Or, so that we can allow u and l to be scalars, + if scalarL, al = sum(a(S1))*l; else al = a(S1)'*l(S1); end + if scalarU, au = sum(a(S2))*u; else au = a(S2)'*u(S2); end + betaEst = (a(S)'*y(S) + au + al -alpha )/(a(S)'*a(S)); + x = projBox( y - betaEst*a ); + otherwise + error( 'Wrong number of arguments.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_conic.m b/proj_conic.m new file mode 100644 index 0000000..fc166a7 --- /dev/null +++ b/proj_conic.m @@ -0,0 +1,48 @@ +function op = proj_conic() +%PROJ_CONIC +% Returns an operator implementing projection onto +% the second order cone, aka (real) Lorentz cone +% aka ice-cream cone +% That is, +% { x : norm( x(1:end-1) , 2) <= x(end) } +% +% The cone is often written as +% { (x,t) : ||x|| <= t } +% so note that in this implementation, "t" is +% inferred from the final coordinate of x. +% +% Contributed by Joself Salmon 2013 + +op = @proj_conic_impl; + +function [ v, x ] = proj_conic_impl( x, t ) +v = 0; +[n,m]=size(x); +sum_part=sqrt( sum(x(1:n-1).^2) ); +xn=(x(n)); + +switch nargin, + case 1, + if nargout == 2 + error( 'This function is not differentiable.' ); + elseif ( (sum_part)>xn ) + v = Inf; + end + case 2, + + if ( ((sum_part) -abs(xn))> 0 ) + x=1/2*(1+xn/sum_part)*[x(1:n-1);sum_part]; + + elseif ( (sum_part)<(xn) ) + x=x; + elseif ( (sum_part)<(-xn) ) + x=zeros(n,m); + + end + + otherwise, + error( 'Not enough arguments.' ); +end +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_l1.m b/proj_l1.m new file mode 100644 index 0000000..8a0e0be --- /dev/null +++ b/proj_l1.m @@ -0,0 +1,42 @@ +function op = proj_l1( q ) +%PROJ_L1 Projection onto the scaled 1-norm ball. +% OP = PROJ_L1( Q ) returns an operator implementing the +% indicator function for the 1-norm ball of radius q, +% { X | norm( X, 1 ) <= q }. Q is optional; if omitted, +% Q=1 is assumed. But if Q is supplied, it must be a positive +% real scalar. +% Dual: prox_linf.m +% See also: prox_linf, prox_l1, proj_linf + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument must be positive.' ); +end +op = @(varargin)proj_l1_q(q, varargin{:} ); + +function [ v, x ] = proj_l1_q( q, x, t ) +v = 0; +switch nargin, +case 2, + if nargout == 2, + error( 'This function is not differentiable.' ); + elseif norm( x(:), 1 ) > q, + v = Inf; + end +case 3, + s = sort(abs(nonzeros(x)),'descend'); + cs = cumsum(s); + % ndx = find( cs - (1:numel(s))' .* [ s(2:end) ; 0 ] >= q, 1 ); + ndx = find( cs - (1:numel(s))' .* [ s(2:end) ; 0 ] >= q+2*eps(q), 1 ); % For stability + if ~isempty( ndx ) + thresh = ( cs(ndx) - q ) / ndx; + x = x .* ( 1 - thresh ./ max( abs(x), thresh ) ); % May divide very small numbers + end +otherwise, + error( 'Not enough arguments.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_l2.m b/proj_l2.m new file mode 100644 index 0000000..c7f7d4e --- /dev/null +++ b/proj_l2.m @@ -0,0 +1,83 @@ +function op = proj_l2( q ) + +%PROJ_L2 Projection onto the scaled 2-norm ball. +% OP = PROJ_L2( Q ) returns an operator implementing the +% indicator function for the 2-norm ball of size q, +% { X | norm( X, 2 ) <= q }. Q is optional; if omitted, +% Q=1 is assumed. But if Q is supplied, it must be a positive +% real scalar. +% Dual: prox_l2.m +% See also: prox_l2.m + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) + error( 'Argument must be a real scalar.' ); +end +if numel(q) == 1 + if q <= 0, error('Argument must be positive'); end + op = @(varargin)proj_l2_q( q, varargin{:} ); +else + if any( abs(q) < 10*eps ), error('Weight "q" must be nonzero'); end + warning('TFOCS:experimental','Using experimental feature of TFOCS'); + + op = @(varargin)proj_l2_qVec( q, varargin{:} ); +end + +function [ v, x ] = proj_l2_q( q, x, t ) +v = 0; +switch nargin, + case 2, + if nargout == 2, + error( 'This function is not differentiable.' ); + elseif norm( x(:), 'fro' ) > q, % GKC fix 2013 (for > 2D arrays) + v = Inf; + end + case 3, + nrm = norm(x(:),'fro'); % fixing, Feb '11, and GKC fix 2013 + if nrm > q + x = x .* ( q / nrm ); + end + otherwise, + error( 'Not enough arguments.' ); +end + +% -- experimental version for when q is a vector -- +function [ v, x ] = proj_l2_qVec( q, x, t ) +v = 0; +switch nargin, + case 2, + if nargout == 2, + error( 'This function is not differentiable.' ); + elseif norm( x./q, 'fro' ) > 1, + v = Inf; + end + case 3, + nrm = norm(x./q,'fro'); + if nrm > 1 + + % We know x is of the form x0./( 1 + lambda*D2 ) + % for some lambda > 0, but we don't have an easy + % way to know what lambda is. So, treat this as + % a 1D minimization problem to find lambda. + D = 1./(q); + D2 = D.^2; + Dx = D.*x; + +% lMax = max( abs(x./D2) )*sqrt(numel(x)); + lMax = 1.2*norm( abs(x./D2),'fro'); % a tighter bound + fmin_opts = optimset( 'TolX', 1e-10 ); +% MaxFunEvals: 500 +% MaxIter: + [lOpt,val,exitflag,output] = fminbnd( @(l) (norm(Dx./(1+l*D2),'fro')-1)^2, 0, lMax,fmin_opts); + if val > 1e-3, error('Proj_l2 failed to converge'); end + x = x./( 1 + lOpt*D2 ); + end + + otherwise, + error( 'Not enough arguments.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_l2group.m b/proj_l2group.m new file mode 100644 index 0000000..9b96d31 --- /dev/null +++ b/proj_l2group.m @@ -0,0 +1,79 @@ +function op = proj_l2group( q,group_indices ) +%PROJ_L2GROUP Projection onto the intersection of scaled 2-norm balls. +% OP = PROJ_L2GROUP( Q, GROUP_INDICES ) returns an operator implementing the +% indicator function for the intersection of 2-norm ball of size q_k, +% i.e. intersection_k B_k +% where B_k = { X | norm( X(indK) ) <= q(k) } +% and indK is indexed by group_indices. Group_indices keeps track +% of the last index in each index set. +% The index sets must be non-overlapping. +% For example, if K=2, and ind1 = 1:10 and ind2 = 11:20, +% then group_indices = [10,20]. +% Q must be a vector of length K or a vector of length N. +% +% With contributions from Joseph Salmon +% +% See also: proj_l2.m + +if isempty(q), q = 1; end +% Make sure q is a column vector +if size(q,1) == 1 && size(q,2) > 1, q = q.'; end +q = expand_q(q,group_indices); +op = @(varargin)proj_l2_q( q,group_indices, varargin{:} ); +end + +function [ v, x ] = proj_l2_q( q,group_indices, x, t ) +v = 0; +nrm=group_norm(x,group_indices); + +switch nargin, + case 3, + if nargout == 2, + error( 'This function is not differentiable.' ); + elseif norm( nrm, 'inf') > q, + v = Inf; + end + case 4, + x = x .* min(1,( q ./ nrm )); + otherwise, + error( 'Not enough arguments.' ); +end +end + + +function y=group_norm(x,group_indices) +% the groups are indexed by their end point + K=length(group_indices); + y=zeros(size(x)); + j=1; % start of the group + for k=1:K + end_point=group_indices(k); + y(j:end_point)=norm(x(j:end_point)); + j=end_point+1; + end +end + +function Q = expand_q(q,group_indices) +% If we have 2 groups, each of size 10 +% (group_indices = [10,20]), then q should +% be a vector of size 20. But it's more convenient +% for the user to pass in a vector of size 2, so this function +% will convert it... +n = group_indices(end); +K = length(group_indices); +if length(q) < n + Q = ones(n,1); + j=1; % start of the group + for k=1:K + end_point=group_indices(k); + Q(j:end_point) = q(k); + j=end_point+1; + end +else + Q = q; +end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_linf.m b/proj_linf.m new file mode 100644 index 0000000..9e72811 --- /dev/null +++ b/proj_linf.m @@ -0,0 +1,36 @@ +function op = proj_linf( q ) + +%PROJ_LINF Projection onto the scaled infinity norm ball. +% OP = PROJ_LINF( Q ) returns an operator implementing the +% indicator function for the infinity norm ball of size q, +% { X | norm( X, Inf ) <= q }. Q is optional; if omitted, +% Q=1 is assumed. But if Q is supplied, it must be a positive +% real scalar. +% Dual: prox_l1.m +% See also: prox_l1, prox_linf, proj_l1 + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument must be positive.' ); +end +op = @(varargin)proj_linf_q( q, varargin{:} ); + +function [ v, x ] = proj_linf_q( q, x, t ) +v = 0; +switch nargin, + case 2, + if nargout == 2, + error( 'This function is not differentiable.' ); + elseif norm( x(:), Inf ) > q, + v = Inf; + end + case 3, + x = x ./ max( 1, abs( x / q ) ); + otherwise, + error( 'Not enough arguments.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_linfl2.m b/proj_linfl2.m new file mode 100644 index 0000000..490a609 --- /dev/null +++ b/proj_linfl2.m @@ -0,0 +1,64 @@ +function op = proj_linfl2( q ) + +%PROJ_LINFL2 Projection of each row onto the scaled l2 norm ball. +% OP = PROJ_LINFL2( Q ) returns an operator implementing the +% indicator function for the set of l2 norm ball of size q, +% { X | for all rows i, norm( X(i,:),2) <= q }. Q is optional; if omitted, +% Q=1 is assumed. But if Q is supplied, it must be a positive +% real scalar. +% Dual: prox_l1l2.m +% See also: prox_l1, prox_linf, proj_l1 + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument must be positive.' ); +end + +% In r2007a and later, we can use bsxfun instead of the spdiags trick +vr=version('-release'); +if str2num(vr(1:4)) >= 2007 + op = @(varargin)proj_linfl2_q_bsxfun( q, varargin{:} ); +else + % the default, using spdiags + op = @(varargin)proj_linfl2_q( q, varargin{:} ); +end + +function [ v, x ] = proj_linfl2_q( q, x, t ) +v = 0; +switch nargin, + case 2, + if nargout == 2, + error( 'This function is not differentiable.' ); + elseif norm( x(:), Inf ) > q, + v = Inf; + end + case 3, + % Compute the norms of the rows + m = size(x,1); + nrms = sqrt( sum( abs(x).^2 , 2 ) ); + % Scale the rows using left diagonal multiplication + x = spdiags( min(1,q./nrms), 0, m, m )*x; + otherwise, + error( 'Not enough arguments.' ); +end + +function [ v, x ] = proj_linfl2_q_bsxfun( q, x, t ) +v = 0; +switch nargin, + case 2, + if nargout == 2, + error( 'This function is not differentiable.' ); + elseif norm( x(:), Inf ) > q, + v = Inf; + end + case 3, + nrms = sqrt( sum( abs(x).^2 , 2 ) ); + bsxfun( @times, x, min(1,q./nrms) ); + otherwise, + error( 'Not enough arguments.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_max.m b/proj_max.m new file mode 100644 index 0000000..b729531 --- /dev/null +++ b/proj_max.m @@ -0,0 +1,36 @@ +function op = proj_max( q ) + +%PROJ_max Projection onto the scaled max-function ball. +% OP = PROJ_MAX( Q ) returns an operator implementing the +% indicator function for the max-function ball of size q, +% { X | max( X(:) ) <= q }. Q is optional; if omitted, +% Q=1 is assumed. But if Q is supplied, it must be a +% real scalar (negative is OK). +% Dual: prox_l1pos.m +% See also: prox_l1pos, prox_linf, proj_l1, proj_linf + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 + error( 'Argument "q" must be a real scalar.' ); +end +op = @(varargin)proj_linf_q( q, varargin{:} ); + +function [ v, x ] = proj_linf_q( q, x, t ) +v = 0; +switch nargin, + case 2, + if nargout == 2, + error( 'This function is not differentiable.' ); + elseif max( x(:) ) > q, + v = Inf; + end + case 3, + x = min( x, q ); + otherwise, + error( 'Not enough arguments.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_maxEig.m b/proj_maxEig.m new file mode 100644 index 0000000..37615ca --- /dev/null +++ b/proj_maxEig.m @@ -0,0 +1,141 @@ +function op = proj_maxEig( q, LARGESCALE ) + +%PROJ_MAXEIG Projection onto the set of matrices with max eigenvalue less than or equal to q +% OP = PROJ_MAXEIG( q ) returns a function that implements the +% indicator for matrices with spectral norm less than q. +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% it must be a positive real scalar. +% This function is similar proj_spectral.m but assumes that +% the input is real symmetric. For positive semi-definite inputs, +% this function is equivalent to proj_spectral +% OP = PROJ_MAXEIG( ..., largescale) will switch to using +% eigs instead of svd or eig, if largescale==true. This is usually +% beneficial for large, sparse variables. +% +% Dual: prox_trace(q) +% (if domain is pos. semidefinite matrices, then prox_trace(q) +% is also the dual, and is more efficient than prox_nuclear(q) ). +% +% See also prox_trace, prox_nuclear, proj_spectral, prox_spectral + +% Sept 1, 2012 +if nargin < 2 || isempty(LARGESCALE) + LARGESCALE = false; +end + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument must be positive.' ); +end +% vectorFunction = proj_linf( q ); % for proj_spectral.m +vectorFunction = proj_max( q ); + +if LARGESCALE + % clear the persistent values: + proj_maxEig_eigs_q(); + op = @(varargin)proj_maxEig_eigs_q( q,vectorFunction, varargin{:} ); +else + op = @(varargin)proj_maxEig_eig_q( q,vectorFunction, varargin{:} ); +end + + +function [ v, X ] = proj_maxEig_eig_q( q,eproj, X, t ) +SP = issparse(X); +v = 0; +if nargin > 3 && t > 0, + if SP, X = full(X); end % svd, eig, and norm require full matrices + [V,D] = eig(X); % just in case X is sparse + [dum,D] = eproj(diag(D),q); % not q*t, since we are just projecting... + X = V*diag(D)*V'; + if SP, X = sparse(X); end +else + nrm = max(eig(X)); + if nrm > q + v = Inf; + end +end + +% --------------- largescale functions: use eigs or svds ----------------------- +function [ v, X ] = proj_maxEig_eigs_q( q,eproj, X, t ) +persistent oldRank +persistent nCalls +persistent V +if nargin == 0, oldRank = []; v = nCalls; nCalls = []; V=[]; return; end +if isempty(nCalls), nCalls = 0; end +SP = issparse(X); +v = 0; +if nargin > 3 && t > 0, + + if isempty(oldRank), K = 10; + else, K = oldRank + 2; + end + [M,N] = size(X); + + ok = false; + opts = []; + opts.tol = 1e-10; + if isreal(X) + opts.issym = true; + SIGMA = 'LA'; % not 'LM' like proj_spectral + else + SIGMA = 'LR'; + end + while ~ok + K = min( [K,M,N] ); + if K > min(M,N)/2 || K > (min(M,N)-2) || min(M,N) < 20 + [V,D] = eig(full((X+X')/2)); + ok = true; + else + [V,D] = eigs( X, K, SIGMA, opts ); + ok = (min(abs(diag(D))) < q) || ( K == min(M,N) ); + end + if ok, break; end + K = 2*K; + if K > 10 + opts.tol = 1e-6; + end + if K > 40 + opts.tol = 1e-4; + end + if K > 100 + opts.tol = 1e-3; + end + end + oldRank = length(find(diag(D) > q)); % no abs here + + [dum,D_proj] = eproj(diag(D),q); + % we want to keep the singular vectors that we haven't discovered + % small = X - V*D*V' + % large = V*D_proj*V' + % and add together to get X - V*(D-Dproj)*V' + + X = X - V*diag(diag(D)-D_proj)*V'; + + if SP, X = sparse(X); end +else +% if SP + opts = struct('tol',1e-8); + if isreal(X) + opts.issym = true; + SIGMA = 'LA'; % not 'LM' like proj_spectral + else + SIGMA = 'LR'; + end + K = 1; + D = eigs( X, K, SIGMA , opts ); + nrm = max(D); +% else +% nrm = max(eig(X)); +% end + if nrm > q + v = Inf; + end +end + + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/proj_nuclear.m b/proj_nuclear.m new file mode 100644 index 0000000..db9ce70 --- /dev/null +++ b/proj_nuclear.m @@ -0,0 +1,56 @@ +function op = proj_nuclear( q ) + +%PROJ_NUCLEAR Projection onto the set of matrices with nuclear norm less than or equal to q. +% OP = PROJ_NUCLEAR( q ) returns a function that implements the +% indicator for matrices with nuclear norm less than q. +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% it must be a positive real scalar. +% This function is like proj_psdUTrace.m but does not assume +% that inputs are square Hermitian positive semidefinite matrices. +% +% This version uses a dense svd decomposition; future versions +% of TFOCS will take advantage of low-rank and/or sparse structure. +% Dual function: prox_spectral.m +% See also prox_spectral, proj_psdUTrace.m, proj_simplex.m, prox_nuclear.m + +% June 26, 2012 + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument must be positive.' ); +end +q = proj_simplex( q ); +op = @(varargin)proj_nuclear_q( q, varargin{:} ); + +function [ v, X ] = proj_nuclear_q( eproj, X, t ) + +VECTORIZE = false; +% Input must be a matrix + +% In proj_psdUTrace.m, we allow vector inputs because +% we can reshape them (since we have square matrices). +% For nuclear norm, inputs can be rectangular matrices, +% so we do not know how to correctly reshape a vector in +% this case. + +% if size(X,1) ~= size(X,2) +% n = sqrt(length(X)); +% X = reshape(X, n, n ); +% VECTORIZE = true; +% end +v = 0; +if nargin > 2 && t > 0, + [U,D,V] = svd(X,'econ'); + [dum,D] = eproj(diag(D),t); + tt = D > 0; + X = U(:,tt)*diag(D(tt))*V(:,tt)'; +% if VECTORIZE, X = X(:); end +elseif any(svd(X)<0), + v = Inf; +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/proj_psd.m b/proj_psd.m new file mode 100644 index 0000000..34e17ec --- /dev/null +++ b/proj_psd.m @@ -0,0 +1,138 @@ +function op = proj_psd( LARGESCALE, isReal, K ) + +% PROJ_PSD Projection onto the positive semidefinite cone. +% OP = PROJ_PSD() returns a function that implements +% the projection onto the semidefinite cone: +% X = argmin_{min(eig(X))>=0} norm(X-Y,'fro') +% +% OP = PROJ_PSD( LARGESCALE ) +% performs the same computation, but in a more efficient +% manner for the case of sparse (and low-rank) matrices +% +% OP = PROJ_PSD( LARGESCALE, isReal ) +% also includes the constraint that X is real-valued if isReal is true. +% +% This function is self-dual. +% See also proj_Rplus.m, the vector analog of this function + +% in the future, we might include this nonconvex version: +% OP = PROJ_PSD( LARGESCALE, isReal, k ) +% only returns at most a rank k matrix +% + +if nargin == 0 || isempty(LARGESCALE), LARGESCALE = false; end +if nargin < 2 || isempty(isReal), isReal = false; end +if nargin < 3, K = Inf; end + +if ~LARGESCALE + op = @(varargin)proj_psd_impl(isReal, varargin{:} ); +else + proj_psd_largescale(); % reset any counters + op = @(varargin)proj_psd_largescale( K, isReal, varargin{:} ); +end + + +function [ v, X ] = proj_psd_impl( isReal, X, t ) +if nargin > 2 && t > 0, + v = 0; + X = full(X+X'); % divide by 2 later + if isReal, X = real(X); end + [V,D]=eig(X); % we don't yet take advantage of sparsity here + D = max(0.5*diag(D),0); + tt = D > 0; + V = bsxfun(@times,V(:,tt),sqrt(D(tt,:))'); + X = V * V'; +else + s = eig(full(X+X'))/2; + if min(s) < -8*eps*max(s), + v = Inf; + else + v = 0; + end +end + + +function [ v, X ] = proj_psd_largescale(Kignore,isReal, X, t ) +% Updated Sept 2012. The restriction to rank K "Kignore" has not been done yet +% (that is nonconvex) +persistent oldRank +persistent nCalls +persistent V +if nargin == 0, oldRank = []; v = nCalls; nCalls = []; V=[]; return; end +if isempty(nCalls), nCalls = 0; end +SP = issparse(X); + +if nargin > 3 && t > 0, + v = 0; + if isempty(oldRank), K = 10; + else, K = oldRank + 2; + end + + [M,N] = size(X); + EIG_TOL = 1e-10; + ok = false; + opts = []; + opts.tol = 1e-10; + if isreal(X) + opts.issym = true; + SIGMA = 'LA'; + else + SIGMA = 'LR'; % largest real part + end + X = (X+X')/2; + if isReal, X = real(X); end + while ~ok + K = min( [K,N] ); + if K > N/2 || K > N-2 || N < 20 + [V,D] = eig(full((X+X')/2)); + ok = true; + else + [V,D] = eigs( X, K, SIGMA, opts ); + ok = (min(real(diag(D))) < EIG_TOL) || ( K == N ); + end + if ok, break; end +% opts.v0 = V(:,1); % starting vector + K = 2*K; +% fprintf('Increasing K from %d to %d\n', K/2,K ); + if K > 10 + opts.tol = 1e-6; + end + if K > 40 + opts.tol = 1e-4; + end + if K > 100 + opts.tol = 1e-3; + end + end + D = real( diag(D) ); + oldRank = length(find( D > EIG_TOL )); + + tt = D > EIG_TOL; + V = bsxfun(@times,V(:,tt),sqrt(D(tt,:))'); + X = V * V'; + if SP, X = sparse(X); end +else + opts.tol = 1e-10; + if isreal(X) + opts.issym = true; + SIGMA = 'SA'; + else + SIGMA = 'SR'; % smallest real part + end + K = 1; % we only want the smallest + X = full(X+X'); % divide by 2 later + if isReal, X = real(X); end + d = eigs(X, K, SIGMA, opts ); + d = real(d)/2; + + if d < -10*eps + v = Inf; + else + v = 0; + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/proj_psdUTrace.m b/proj_psdUTrace.m new file mode 100644 index 0000000..02199e0 --- /dev/null +++ b/proj_psdUTrace.m @@ -0,0 +1,193 @@ +function op = proj_psdUTrace( q, LARGESCALE, force_real, maxK ) + +%PROJ_PSDUTRACE Projection onto the positive semidefinite cone with fixed trace. +% OP = PROJ_PSDUTRACE( q ) returns a function that implements the +% indicator for the cone of positive semidefinite (PSD) matrices with +% fixed trace: { X | min(eig(X+X'))>=0, trace(0.5*(X+X'))<=q +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% it must be a positive real scalar. +% +% OP = PROJ_PSDUTRACE( q, LARGESCALE) will use a Lanczos-based Eigenvalue +% decomposition if LARGESCALE==true, eitherwise it uses a dense +% matrix decomposition +% +% OP = PROJ_PSDUTRACE( q, LARGESCALE, forceReal ) will also include the +% constraint that X is a real matrix if forceReal is true. +% +% OP = PROJ_PSDUTRACE( q, LARGESCALE, forceReal, maxK ) will only +% use the Lanczos-solver if it expects fewer than maxK eigenvalues +% +% CALLS = PROJ_PSDUTRACE( 'reset' ) +% resets the internal counter and returns the number of function calls +% +% This version uses a dense eigenvalue decomposition; future versions +% of TFOCS will take advantage of low-rank and/or sparse structure. +% +% If the input to the operator is a vector of size n^2 x 1, it will +% be automatically reshaped to a n x n matrix. In this case, +% the output will also be of length n^2 x 1 and not n x n. +% +% Note: proj_simplex.m (the vector-analog of this function) +% Duals: the dual function is prox_maxEig, which also requires +% PSD inputs. The function prox_spectral(q,'sym') is also equivalent +% to prox_maxEig if given a PSD input. +% See also prox_maxEig, prox_spectral, proj_simplex + +% Feb 15 2013, adding support for eigs calculations + + +if nargin == 1 && strcmpi(q,'reset') + op = prox_trace_impl; + return; +end + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument must be positive.' ); +end +if nargin < 2, LARGESCALE = []; end +if nargin < 3 + force_real = false; +end +if nargin < 4 || isempty(maxK) + maxK = 100; +end +if ~isempty( LARGESCALE ) && LARGESCALE + op = @(varargin)proj_psdUTrace_q_eigs( q, maxK, force_real, varargin{:} ); +else + op = @(varargin)proj_psdUTrace_q( q, force_real, varargin{:} ); +end + +function [ v, X ] = proj_psdUTrace_q_eigs( lambda, maxK, force_real, X, t ) +persistent oldRank +persistent nCalls +persistent V +if nargin == 0, oldRank = []; v = nCalls; nCalls = []; V=[]; return; end +if isempty(nCalls), nCalls = 0; end +if isempty(maxK), maxK = size(X,1); end + + +VECTORIZE = false; +if size(X,1) ~= size(X,2) + %error('proj_psdUTrace requires a square matrix as input'); + n = sqrt(length(X)); + X = reshape(X, n, n ); + VECTORIZE = true; +end +v = 0; +X = full(0.5*(X+X')); % added 'full' Sept 5 2012 +if force_real % added Nov 23 2012 + X = real(X); +end +if nargin > 4 && t > 0 + + opts = []; + opts.tol = 1e-10; + if force_real + opts.issym = true; + SIGMA = 'LA'; % get largest eigenvalues (NOT in magnitude) + else + SIGMA = 'LR'; % should be real anyhow (to 1e-10). get largest real part + end + if isempty(oldRank), K = 10; + else, K = oldRank + 2; + end + N = size(X,1); + ok = false; + ctr = 0; + FEASIBLE = false; + while ~ok + ctr = ctr + 1; + K = min( K, N ); + if K > N/2 || K > maxK + [V,D] = eig(X); ok = true; + D = diag(D); + break; + end +% opts.tol = min(max(opts.tol,1e-10),1e-7); + + [V,D] = eigs( X, K,SIGMA, opts ); + D = diag(D); + delta = ( sum(D) - lambda ) / K; + ok = min(D) < delta; + + if ok, break; end + + % Can we do an early return? maybe we are already feasible + if sum(D) + (N-K-1)*min(D) < lambda +% disp('Point is feasible! exiting'); + FEASIBLE = true; + break; + end + + % otherwise, increase K + + K = 2*K; + if K > 10 + opts.tol = 1e-6; + end + if K > 40 + opts.tol = 1e-4; + end + if K > 100 + opts.tol = 1e-3; + end + end + if FEASIBLE + oldRank = N; + else + smplx = proj_simplex(lambda); + [dum,D] = smplx(D,1); + spprt = find(D); + D = diag(D(spprt)); + V = V(:,spprt); + X = V*D*V'; + oldRank = length(spprt); + end + + + nCalls = nCalls + 1; + + if VECTORIZE, X = X(:); end +elseif any(eig(X)<0) || trace(X) > lambda + v = Inf; +end + + + +function [ v, X ] = proj_psdUTrace_q( lambda, force_real, X, t ) +persistent nCalls +if nargin == 0, v = nCalls; nCalls = []; return; end +if isempty(nCalls), nCalls = 0; end + +eproj = proj_simplex( lambda ); + +VECTORIZE = false; +if size(X,1) ~= size(X,2) + %error('proj_psdUTrace requires a square matrix as input'); + n = sqrt(length(X)); + X = reshape(X, n, n ); + VECTORIZE = true; +end +v = 0; +X = full(0.5*(X+X')); % added 'full' Sept 5 2012 +if force_real % added Nov 23 2012 + X = real(X); +end +if nargin > 3 && t > 0, + nCalls = nCalls + 1; + [V,D]=eig(X); + [dum,D] = eproj(diag(D),t); + tt = D > 0; + V = bsxfun(@times,V(:,tt),sqrt(D(tt,:))'); + X = V * V'; + if VECTORIZE, X = X(:); end +elseif any(eig(X)<0) || trace(X) > lambda + v = Inf; +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/proj_simplex.m b/proj_simplex.m new file mode 100644 index 0000000..ccbe2df --- /dev/null +++ b/proj_simplex.m @@ -0,0 +1,61 @@ +function op = proj_simplex( q ) + +%PROJ_SIMPLEX Projection onto the simplex. +% OP = PROJ_SIMPLEX( Q ) returns an nonsmooth function that +% represents the scaled simplex { x | x >= 0, sum(x) <= q }. +% Q is optional; if not supplied, it defaults to 1. If it is +% supplied, it must be a real positive scalar. +% +% See also proj_psdUTrace.m (the matrix-analog of this function) + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument must be positive.' ); +end +op = @(varargin)proj_simplex_q( q,varargin{:} ); + +function [ v, x ] = proj_simplex_q( q, x, t ) +v = 0; + +% We have two options when input x is a matrix: +% Does the user want to treat it as x(:), i.e. "vectorize" it ? +% Or does the user want to treat each column separately? +% Most other functions (e.g. l1, linf) treat it as x(:) +% so that will be the default. However, we leave it +% as a hard-coded option so that the user can change it +% if they want. +VECTORIZE = true; + +if nargin > 2 && t > 0, + if any( x(:) < 0 ) || any( sum( x ) > q ), + if VECTORIZE + s = sort( x(:), 'descend' ); + else + s = sort( x, 'descend' ); + end + if q < eps(s(1)) + error('Input is scaled so large compared to q that accurate computations are difficult'); + % since then cs(1) = s(1) - q is not even guaranteed to be + % smaller than q ! + end + if size(x,2) == 1 || VECTORIZE + cs = ( cumsum(s) - q ) ./ ( 1 : numel(s) )'; + ndx = nnz( s > cs ); + x = max( x - cs(ndx), 0 ); + else + % vectorized for several columns + cs = diag( 1./( 1 : size(s,1) ) )*( cumsum(s) - q ); + ndx = sum( s > cs ); + ndx = sub2ind( size(x), ndx, 1:size(x,2) ); + x = max( x - repmat(cs(ndx),size(x,1),1), 0 ); + end + end +% Sept 6 2012, adding factor of 100 in the line below: +elseif any( x(:) < 0 ) || any( abs( sum(x) / q - 1 ) > sqrt(numel(x)) * 100 *eps ) + v = Inf; +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_singleAffine.m b/proj_singleAffine.m new file mode 100644 index 0000000..eddc05f --- /dev/null +++ b/proj_singleAffine.m @@ -0,0 +1,50 @@ +function op = proj_singleAffine( a, beta, ineq ) +%PROJ_SINGLEAFFINE(a,beta) Projection onto the affine set { x : a'*x == beta } +% The parameter a may be a vector or matrix +% of the same size as x. By default, beta is 0 +%PROJ_SINGLEAFFINE(a,beta,ineq) +% Projection onto the affine set { x : a'*x >= beta } +% if ineq == 1, and onto { x : a'*x <= beta } if ineq == -1 +% Setting ineq == 0 (default) uses { x : a'*x == beta } +% +% For matrix variables, this interprets a'*x as a(:)'*x(:), e.g., +% the standard dot product +% +% OP = PROJ_SINGLEAFFINE returns an implementation of this projection. +% +% For more than one affine constraint, see proj_affine.m +% +% See also proj_boxAffine and proj_affine + +error(nargchk(1,3,nargin)); +if nargin < 2, beta = 0; end +if isempty(beta), beta = 0; end +if nargin < 3 || isempty(ineq), ineq = 0; end +op = @(varargin) proj_affine_internal(a,beta,ineq, varargin{:} ); + + +function [v,x] = proj_affine_internal( a, beta, ineq, y, t ) +v = 0; +switch nargin + case 4 + if nargout == 2 + error('This function is not differentiable.'); + end + % Check if we are feasible + if abs( tfocs_dot(y,a) - beta ) > 1e-13 + v = Inf; + end + case 5 + dt = tfocs_dot(a,y); + if (ineq==0) || ( ineq==1 && dt < beta ) || (ineq==-1 && dt > beta) + x = y - ( dt - beta )/tfocs_dot(a,a) * a; + else + x = y; + end + otherwise + error( 'Wrong number of arguments.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/proj_spectral.m b/proj_spectral.m new file mode 100644 index 0000000..f561935 --- /dev/null +++ b/proj_spectral.m @@ -0,0 +1,226 @@ +function op = proj_spectral( q, SYM_FLAG, LARGESCALE ) + +%PROJ_SPECTRAL Projection onto the set of matrices with spectral norm less than or equal to q +% OP = PROJ_SPECTRAL( q ) returns a function that implements the +% indicator for matrices with spectral norm less than q. +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% it must be a positive real scalar. +% This function is like proj_psdUTrace.m but does not assume +% that inputs are square Hermitian positive semidefinite matrices. +% OP = PROJ_SPECTRAL( q, 'sym' ) or OP = PROJ_SPECTRAL( 'eig' ) +% will instruct the code that the matrix is Hermitian and +% therefore the relations between singular- and eigen-values +% are well known, and the code will use the more efficient +% eigenvalue decomposition (instead of the SVD). +% OP = PROJ_SPECTRAL( ..., largescale) will switch to using +% svds (or PROPACK, if installed and in the path) or eigs +% instead of svd or eig, if largescale==true. This is usually +% beneficial for large, sparse variables. +% +% Dual: prox_nuclear(q) +% (if domain is pos. semidefinite matrices, then prox_trace(q) +% is also the dual, and is more efficient than prox_nuclear(q) ). +% +% See also prox_nuclear, proj_linf, proj_nuclear, prox_spectral, proj_maxEig + +% Sept 1, 2012 +if nargin < 3 || isempty(LARGESCALE) + LARGESCALE = false; +end + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument must be positive.' ); +end +vectorFunction = proj_linf( q ); +if nargin >= 2 && ( ... + ~isempty(strfind(lower(SYM_FLAG),'eig')) || ... + ~isempty(strfind(lower(SYM_FLAG),'sym')) ) + if LARGESCALE + % clear the persistent values: + proj_spectral_eigs_q(); + op = @(varargin)proj_spectral_eigs_q( q,vectorFunction, varargin{:} ); + else + op = @(varargin)proj_spectral_eig_q( q,vectorFunction, varargin{:} ); + end +else + if LARGESCALE + % clear the persistent values: + proj_spectral_svds_q(); + op = @(varargin)proj_spectral_svds_q( q,vectorFunction, varargin{:} ); + else + op = @(varargin)proj_spectral_svd_q( q,vectorFunction, varargin{:} ); + end +end + +function [ v, X ] = proj_spectral_svd_q( q,eproj, X, t ) +sx = size( X ); +SP = issparse(X); +if length( sx ) > 2, + X = reshape( X, prod(sx(1:end-1)), sx(end) ); +end +v = 0; +if nargin > 3 && t > 0, + if SP, X = full(X); end % svd, eig, and norm require full matrices + [U,D,V] = svd( X, 'econ' ); + [dum,D] = eproj(diag(D),q); % not q*t, since we are just projecting... + X = U*diag(D)*V'; + if SP, X = sparse(X); end + X = reshape( X, sx ); +else + if SP, + [nrm,cnt] = normest(X,1e-3); + else + nrm = norm(X); + end + if nrm > q + v = Inf; + end +end + +function [ v, X ] = proj_spectral_eig_q( q,eproj, X, t ) +SP = issparse(X); +v = 0; +if nargin > 3 && t > 0, + if SP, X = full(X); end % svd, eig, and norm require full matrices + [V,D] = eig(X); % just in case X is sparse + [dum,D] = eproj(diag(D),q); % not q*t, since we are just projecting... + X = V*diag(D)*V'; + if SP, X = sparse(X); end +else + if SP, [nrm,cnt] = normest(X,1e-3); + else nrm = norm(X); + end + if nrm > q + v = Inf; + end +end + +% --------------- largescale functions: use eigs or svds ----------------------- +function [ v, X ] = proj_spectral_eigs_q( q,eproj, X, t ) +persistent oldRank +persistent nCalls +persistent V +if nargin == 0, oldRank = []; v = nCalls; nCalls = []; V=[]; return; end +if isempty(nCalls), nCalls = 0; end +SP = issparse(X); +v = 0; +if nargin > 3 && t > 0, + + if isempty(oldRank), K = 10; + else, K = oldRank + 2; + end + [M,N] = size(X); + + ok = false; + opts = []; + opts.tol = 1e-10; + if isreal(X) + opts.issym = true; + end + while ~ok + K = min( [K,M,N] ); + [V,D] = eigs( X, K, 'LM', opts ); + ok = (min(abs(diag(D))) < q) || ( K == min(M,N) ); + if ok, break; end + K = 2*K; + if K > 10 + opts.tol = 1e-6; + end + if K > 40 + opts.tol = 1e-4; + end + if K > 100 + opts.tol = 1e-3; + end + if K > min(M,N)/2 + [V,D] = eig(full((X+X')/2)); + ok = true; + end + end + oldRank = length(find(abs(diag(D)) > q)); + + [dum,D_proj] = eproj(diag(D),q); + % we want to keep the singular vectors that we haven't discovered + % small = X - V*D*V' + % large = V*D_proj*V' + % and add together to get X - V*(D-Dproj)*V' + + X = X - V*diag(diag(D)-D_proj)*V'; + + if SP, X = sparse(X); end +else + if SP, [nrm,cnt] = normest(X,1e-3); + else nrm = norm(X); + end + if nrm > q + v = Inf; + end +end + + +function [ v, X ] = proj_spectral_svds_q( q,eproj, X, t ) +persistent oldRank +persistent nCalls +persistent V +if nargin == 0, oldRank = []; v = nCalls; nCalls = []; V=[]; return; end +if isempty(nCalls), nCalls = 0; end +SP = issparse(X); +v = 0; +if nargin > 3 && t > 0, + + if isempty(oldRank), K = 10; + else, K = oldRank + 2; + end + [M,N] = size(X); + + ok = false; + opts = []; + opts.tol = 1e-10; % the default in svds + opt = []; + opt.eta = eps; % makes compute_int slow + opt.delta = 10*opt.eta; + while ~ok + K = min( [K,M,N] ); + if exist('lansvd','file') + [U,D,V] = lansvd(X,K,'L',opt ); + else + [U,D,V] = svds(X,K,'L',opts); + end + ok = (min(abs(diag(D))) < q) || ( K == min(M,N) ); + if ok, break; end + K = 2*K; + if K > 10 + opts.tol = 1e-6; + end + if K > 40 + opts.tol = 1e-4; + end + if K > 100 + opts.tol = 1e-3; + end + if K > min(M,N)/2 + [U,D,V] = svd( full(X), 'econ' ); + ok = true; + end + end + oldRank = length(find(abs(diag(D)) > q)); + + [dum,D_proj] = eproj(diag(D),q); + X = X - U*diag(diag(D)-D_proj)*V'; + + if SP, X = sparse(X); end +else + if SP, [nrm,cnt] = normest(X,1e-3); + else nrm = norm(X); + end + if nrm > q + v = Inf; + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/prox_0.m b/prox_0.m new file mode 100644 index 0000000..ad3c93b --- /dev/null +++ b/prox_0.m @@ -0,0 +1,13 @@ +function op = prox_0 + +%PROX_0 The zero proximity function: +% OP = PROX_0 returns an implementation of the function OP(X) = 0. Use +% this function to specify a model with no nonsmooth component. It is +% identical to both PROJ_Rn and SMOOTH_CONSTANT( 0 ). +% Dual: proj_0.m + +op = smooth_constant( 0 ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_Ol1.m b/prox_Ol1.m new file mode 100644 index 0000000..738d983 --- /dev/null +++ b/prox_Ol1.m @@ -0,0 +1,87 @@ +function op = prox_Ol1( lambda ) +%PROX_OL1 Ordered L1 norm. +% OP = PROX_L1( lambda ) implements the nonsmooth function +% OP(X) = sum(lambda.*sort(abs(X),'descend')) +% where lambda is strictly positive and sorted in decreasing order, +% in which case this function is a norm (and hence convex). +% If lambda is a scalar, it will be expanded to all elements, but in this +% case it is equivalent to the (scaled) usual l1 norm. +% +% Notice: this function uses mex files. Some pre-compiled binaries +% for common systems are included; if these do not work for you, +% then please install yourself. In the mexFiles/ subdirectory, +% run the file "makeMex.m" +% Reference: +% http://www-stat.stanford.edu/~candes/OrderedL1/ +% "Statistical Estimation and Testing via the Ordered L1 Norm" +% by M. Bogdan, E. van den Berg, W. Su, and E. J. Candès +% 2013 +% +% See also prox_l1.m, makeMex.m + +if nargin == 0, + lambda = 1; +elseif ~isnumeric( lambda ) || ~isreal( lambda ) || any( lambda <= 0 ) + error( 'Argument lambda must have all positive entries.' ); +end +if ~issorted(flipud(lambda(:))) + error( 'Argument lambda must be sorted in decreasing order.'); +end +if numel(lambda)==1 + warning('TFOCS:prox_OL1','When lambda is a scalar, we recommend prox_l1.m instead pf prox_OL1.m'); +end + + +% The mex file is in the child directory mexFiles/ +% Check for its existence. First, add the right paths +addpath(fullfile(tfocs_where,'mexFiles')); +if 3 ~= exist('proxAdaptiveL1Mex','file') + makeMex; + % check that it worked + if 3 ~= exist('proxAdaptiveL1Mex','file') + disp('Compilation of mex files for prox_OL1.m failed; please report this error'); + end +end + +f = @(x) sum( lambda(:) .* sort(abs(x(:)), 'descend') ); +prox_f = @(x,t) proxOrderedL1(x,t.*lambda); + +op = tfocs_prox( f, prox_f , 'vector' ); % Allow vector stepsizes + + +end + +% -- subroutines -- +function x = proxOrderedL1(y,lambda) + % Normalization + lambda = lambda(:); + y = y(:); + sgn = sign(y); + [y,idx] = sort(abs(y),'descend'); + + % Simplify the problem + k = find(y > lambda,1,'last'); + + % Compute solution and re-normalize + n = numel(y); + x = zeros(n,1); + + if (~isempty(k)) + v1 = y(1:k); + if numel(lambda) > 1 + v2 = lambda(1:k); + else + v2 = lambda*ones(k,1); % if lambda is a scalar, implicity make it lambda*ones(size(y)) + end + v = proxAdaptiveL1Mex(v1,v2); + x(idx(1:k)) = v; + end + + % Restore signs + x = sgn .* x; +end + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_boxDual.m b/prox_boxDual.m new file mode 100644 index 0000000..f5c03f1 --- /dev/null +++ b/prox_boxDual.m @@ -0,0 +1,82 @@ +function op = prox_boxDual(l,u,scale) +%PROX_BOXDUAL Dual function of box indicator function { l <= x <= u } +%PROX_BOXDUAL(l,u) Dual function of box indicator function { l <= x <= u } +% If l or u is the empty matrix [], then the constraint is not +% enforced (e.g. PROJ_BOXDUAL([],1) is the set { x <= 1 }, +% and PROJ_BOXDUAL(0) is the set { 0 <= x } ) +% The parameters l and u may also be vectors or matrixes +% of the same size as x. +% +% OP = PROJ_BOXDUAL returns an implementation of this projection. +% +% ... = PROJ_BOXDUAL( l, u, -1 ) +% will return an implementation of the projection composed +% with the operator x --> -x. This is the form that TFOCS +% expects for the "conjnegeF" term. +% +% See also proj_box, proj_Rplus + + +error(nargchk(1,3,nargin)); +if nargin < 3, scale = 1; +else + if ~isscalar(scale) || ( scale ~= 1 && scale ~= -1 ) + error('"scale" must be either +1 or -1'); + end +end +if nargin < 2, u = []; end +% check that l <= u +if ~isempty(l) && ~isempty(u) + if ~all( l <= u ) + error('for box constraints, need l <= u '); + end +end +if isempty(l), l = -Inf; end +if isempty(u), u = Inf; end + +op = @(varargin)proj_RplusDual_impl(l,u,scale,varargin{:}); + +function [ v, xOut ] = proj_RplusDual_impl( l, u, scale, x, t ) + if scale ~= 1 + x = scale*x; + end +switch nargin, + case 4, + if nargout == 2, + error( 'This function is not differentiable.' ); + end + v = sum(sum(max( l.*x, u.*x ))); + case 5, + xOut = zeros( size(x) ); + if ~isempty(l) + indx1 = find( x < t*l ); + if isscalar(l) + xOut( indx1 ) = x( indx1 ) - t*l; + else + xOut( indx1 ) = x( indx1 ) - t*l( indx1 ); + end + end + if ~isempty(u) + indx2 = find( x > t*u ); + if isscalar(u) + xOut( indx2 ) = x( indx2 ) - t*u; + else + xOut( indx2 ) = x( indx2 ) - t*u( indx2 ); + end + end + % and implicity, if l/t < x < u/t, then xOut is 0 + + if scale ~= 1 + xOut = scale*xOut; + end + + v = sum(sum( max( l.*xOut, u.*xOut ) )); +% v = sum(sum( max( [l.*xOut, u.*xOut] ) )); + otherwise, + error( 'Wrong number of arguments.' ); +end + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_dualize.m b/prox_dualize.m new file mode 100644 index 0000000..1004ba1 --- /dev/null +++ b/prox_dualize.m @@ -0,0 +1,94 @@ +function op = prox_dualize( dualProx, NEG ) +%PROX_DUALIZE Define a proximity function by its dual +% OP = PROX_DUALIZE( dualOp ) returns an operator implementing the +% dual of the function dualProx. You can verify they are duals +% via test_proxPair( dualOp, OP ). +% +% OP = PROX_DUALIZE( dualOp, 'neg' ) +% OP = PROX_DUALIZE( dualOp, 'negative' ) +% will return the scaled dual of dualOp; that is, +% dualOp(x) and OP(-x) are duals. +% The negative is useful because this is the version TFOCS +% expects for the SCD formulation. +% For 1-homogenous functions (e.g. norms), this has no effect, +% since ||x|| = ||-x||. +% +% Warning: if you can calculate the dual function explicitly, +% it is likely more computationally efficient to do so, rather +% than rely on this code. This code requires some tricks, some +% of which can sometimes be expensive; also, it will call dualOp +% so it is at least as slow as dualOp; and it may require high +% precision from dualOp. This code can break down if dualOp +% is not numerically stable. + +if nargin < 2, NEG = ''; end +if strcmpi(NEG,'neg') || strcmpi(NEG,'negative') + op = @(varargin)dualize(dualProx,-1,varargin{:} ); +else + op = @(varargin)dualize(dualProx,1,varargin{:} ); +end + +function [ v, x ] = dualize( dualProx, scale, x, t ) +vec = @(x) x(:); +myDot = @(x,y) x(:)'*y(:); +if scale == -1 + x = -x; +end +switch nargin, + case 3 + if nargout == 2, + error( 'This function is not differentiable.' ); + else + % This case is a bit tricky... + % If the function is non-differentiable, then standard exact + % penalty function results tell us that for a sufficiently + % small stepsize, we can remove the effect of smoothing. + % In other cases, we don't have an exact value, but we hope this + % is a reasonable approximation. + +% t = 1e-15; +% [~,x2] = dualProx( x/t, 1/t ); +% v = myDot(x,x2) - dualProx( x2 ); + + % However, some functions break down when 1/t is huge + % So we will slowly decrease it + vOld = Inf; + t = 1e-5; + ok = false; + iter = 0; + while ~ok && t > eps + [~,x2] = dualProx( x/t, 1/t ); + v = myDot(x,x2) - dualProx( x2 ); + if abs(v-vOld)/max( 1e-10, abs(v) ) < 1e-4 + % due to exact penalty, we expect that + % for t < t_cutoff, v=vOld up to machine accuracy + ok = true; + else + t = t/10; + vOld = v; + iter = iter + 1; + %fprintf('%d and v is %.2e\n', iter, v ); + end + end + + end + + case 4 + % This is exact. + [ignore,x2] = dualProx( x/t, 1/t ); + x1 = x - t*x2; % Moreau's identity, equation (8.1) in the user guide + v = myDot(x1,x2) - dualProx( x2 ); + + % If we think it is an indicator function, then round down to zero: + if abs(v) < 100*eps, + v = 0; + end + x = scale*x1; + + otherwise, + error( 'Not enough arguments.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_hinge.m b/prox_hinge.m new file mode 100644 index 0000000..e64f77b --- /dev/null +++ b/prox_hinge.m @@ -0,0 +1,76 @@ +function op = prox_hinge( q , r, y) + +%PROX_HINGE Hinge-loss function. +% OP = PROX_HINGE( q , r, y ) implements the nonsmooth function +% OP(X) = q * sum( max( r - y.*x, 0 ) ). +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% then it must be a positive real scalar. +% R is also optional; if omitted, R = 1 is assumed. R may be any real number. +% Y is also optional; if omitted, Y = 1 is assumed. Y may be any scalar +% or vector of the same size as X +% Dual: prox_hingeDual.m +% +% See also PROX_HINGEDUAL + +if nargin < 3 + y = []; +elseif ~isempty(y) && ( ~isnumeric(y) || ~isreal(y) ) + error( 'Argument 3 must be a real vector'); +end +if nargin < 2 + r = 1; +elseif ~isnumeric( r ) || ~isreal( r ) %|| numel( r ) ~= 1 + error( 'Argument 2 must be real.' ); +end +if nargin < 1 + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument 1 must be positive.' ); +end + +if isempty(y) + op = tfocs_prox( @(x)hinge(x,r,q), @(x,t)prox_hinge(x,t,r,q) ); +else + ry = r./y; + op = tfocs_prox( @(x)hingeY(x,r,q,y), @(x,t)prox_hingeY(x,t,r,q,ry,y) ); +end + + +% -- the actual functions -- + +function v = hingeY(x,r,q,y) + if ~isscalar(r), assert( numel(r) == numel(x),'r is wrong size' ); end + if ~isscalar(y), assert( numel(y) == numel(x),'y is wrong size'); end + v = q*sum( max( r(:) - y(:).*x(:), 0 ) ); +end +function v = hinge(x,r,q) + if ~isscalar(r), assert( numel(r) == numel(x),'r is wrong size' ); end + v = q*sum( max( r(:) - x(:), 0 ) ); +end + + +% PROX_F( Y, t ) = argmin_X F(X) + 1/(2*t)*|| X - Y ||^2 +function x = prox_hinge(x,t,r,q) + tq = t * q; + x = r + (x-r).*( x > r ) + (x + tq - r).*( x + tq < r ); +end +%{ + in the q = r = t = 1 case, the prox is: +prox(x) = { x, if x > 1 + 1, if x <= 1, and x > 0 + x + q, if x <= 0 +%} + +function x = prox_hingeY(x,t,r,q,ry,y) + tq = t * q; + x = ry + (x-ry).*( y.*x > r ) + (x + tq*y - ry).*( y.*(x + tq*y) < r ); +end + + +end + +% Added Feb, 2011; modified Dec, 2011 + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_hingeDual.m b/prox_hingeDual.m new file mode 100644 index 0000000..cb97d19 --- /dev/null +++ b/prox_hingeDual.m @@ -0,0 +1,97 @@ +function op = prox_hingeDual( q , r, y) + +%PROX_HINGEDUAL Dual function of the Hinge-loss function. +% OP = PROX_HINGEDUAL( q , r , y) implements the nonsmooth function +% that is dual to the Hinge-loss function f, where +% f(x) = q * sum( max( r - y.*x, 0 ) ). +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% then it must be a positive real scalar. +% R is also optional; if omitted, R = 1 is assumed. R may be any real scalar. +% +% There is a simple form for the dual and its proximity operator. +% In the case q = r = 1, the dual is: +% f^*(y) = { +y, if y >= -1 and y <= 0 +% { +Inf, otherwise +% +% See also PROX_HINGE +% +% Note: if the primal is PROX_HINGE( q, r, y ) +% then TFOCS expects conjnegF to be PROX_HINGEDUAL( q, r, -y) +% (the -y is because the conjugate should be composed +% with the function x --> -x ) + +if nargin < 3 + y = []; +elseif ~isempty(y) && ( ~isnumeric(y) || ~isreal(y) ) + error( 'Argument 3 must be a real vector'); +end +if nargin < 2 || isempty(r) + r = 1; +elseif ~isnumeric( r ) || ~isreal( r ) %|| numel( r ) ~= 1 + error( 'Argument must be real.' ); +end +if nargin < 1 || isempty(q) + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument must be positive.' ); +end + +if isempty(y) + if isscalar(r) + sumXR = @(x) r*sum(x); + else + sumXR = @(x) x'*r; + end + op = tfocs_prox( @(x)hingeDual(x,q,sumXR), @(x,t)prox_hingeDual(x,t,r,q) ); +else + ry = r./y; + qy = -q.*abs(y); + sy = sign(y); + if isscalar(r) && isscalar( y ) + sumXR = @(x) ry*sum(x); + else + sumXR = @(x) x'*ry; + end + op = tfocs_prox( @(x)hingeDualY(x,sy,qy,sumXR), @(x,t)prox_hingeDualY(x,t,sy,ry,qy) ); +end + + +function v = hingeDual(x,q,sumXR) + feasible = ( x >= -q & x <= 0 ); + if any( ~feasible ) + v = Inf; + return; + end +% v = sum( x*r ); + v = sumXR( x ); +% v = sum( x*r.*( feasible ) + Inf.*( ~feasible ) ); % 0*Inf leads to NaN. bad! +end + +function v = hingeDualY(x,sy,qy,sumXR) +% feasible = ( sign(y).*x >= -q.*abs(y) & sign(y).*x <= 0 ); + feasible = ( sy.*x >= qy & sy.*x <= 0 ); % using precomputed vectors + if any( ~feasible ) + v = Inf; + return; + end +% v = sum( x.*ry ); + v = sumXR(x); +end + + +% PROX_F( Y, t ) = argmin_X F(X) + 1/(2*t)*|| X - Y ||^2 +function x = prox_hingeDual(x,t,r,q) + x = max( min( x - t*r, 0), -q ); +end +function x = prox_hingeDualY(x,t,sy,ry,qy) + x = sy.*max( min( sy.*(x - t*ry), 0), qy ); +end + + +end + +% Added Feb, 2011; support for y added Dec 2011 + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_l1.m b/prox_l1.m new file mode 100644 index 0000000..23f18cd --- /dev/null +++ b/prox_l1.m @@ -0,0 +1,54 @@ +function op = prox_l1( q ) + +%PROX_L1 L1 norm. +% OP = PROX_L1( q ) implements the nonsmooth function +% OP(X) = norm(q.*X,1). +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% then it must be a positive real scalar (or must be same size as X). +% Dual: proj_linf.m + +% Update Feb 2011, allowing q to be a vector +% Update Mar 2012, allow stepsize to be a vector + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || any( q < 0 ) || all(q==0) %|| numel( q ) ~= 1 + error( 'Argument must be positive.' ); +end + +% The following commented code works fine in Matlab, +% but doesn't work in Octave due to different conventions on nesting +% functions. We keep in in the comments since it may be useful +% as an example of building a prox function + +% op = tfocs_prox( @f, @prox_f , 'vector' ); % Allow vector stepsizes +% function v = f(x) +% v = norm( q(:).*x(:), 1 ); +% end +% function x = prox_f(x,t) +% tq = t .* q; % March 2012, allowing vectorized stepsizes +% s = 1 - min( tq./abs(x), 1 ); +% x = x .* s; +% end +% end % end of main file + +% This is Matlab and Octave compatible code +op = tfocs_prox( @(x)f(q,x), @(x,t)prox_f(q,x,t) , 'vector' ); +end + +% These are now subroutines, that are NOT in the same scope +function v = f(qq,x) + v = norm( qq(:).*x(:), 1 ); +end + +function x = prox_f(qq,x,t) + tq = t .* qq; % March 2012, allowing vectorized stepsizes + s = 1 - min( tq./abs(x), 1 ); + x = x .* s; +end + + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_l1l2.m b/prox_l1l2.m new file mode 100644 index 0000000..3a69952 --- /dev/null +++ b/prox_l1l2.m @@ -0,0 +1,66 @@ +function op = prox_l1l2( q ) + +%PROX_L1L2 L1-L2 block norm: sum of L2 norms of rows. +% OP = PROX_L1L2( q ) implements the nonsmooth function +% OP(X) = q * sum_{i=1:m} norm(X(i,:),2) +% where X is a m x n matrix. If n = 1, this is equivalent +% to PROX_L1 +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% then it must be positive and real. +% If Q is a vector, it must be m x 1, and in this case, +% the weighted norm OP(X) = sum_{i} Q(i)*norm(X(i,:),2) +% is calculated. +% +% Dual: proj_linfl2.m +% See also proj_linfl2.m + +if nargin == 0, + q = 1; +% elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, +elseif ~isnumeric( q ) || ~isreal( q ) || any(q <= 0), + error( 'Argument must be positive.' ); +end +% In r2007a and later, we can use bsxfun instead of the spdiags trick +if exist('OCTAVE_VERSION','builtin') + vr = '2010'; +else + vr=version('-release'); +end +if str2num(vr(1:4)) >= 2007 + op = tfocs_prox( @(x)f(q,x), @(x,t)prox_f_bsxfun(q,x,t) ); +else + % the default, using spdiags + op = tfocs_prox( @(x)f(q,x), @(x,t)prox_f(q,x,t) ); +end + +function v = f(q,x) + if numel(q) ~= 1 && size(q,1) ~= size(x,1) + error('Weight must be a scalar or a column vector'); + end + v = sum( q.* sqrt(sum(x.^2,2)) ); +end + +function x = prox_f(q,x,t) + if nargin < 3, + error( 'Not enough arguments.' ); + end + % v = sqrt( tfocs_normsq( x ) ); + v = sqrt( sum(x.^2,2) ); + s = 1 - 1 ./ max( v ./ ( t .* q ), 1 ); + m = length(s); + x = spdiags(s,0,m,m)*x; +end +function x = prox_f_bsxfun(q,x,t) + if nargin < 3, + error( 'Not enough arguments.' ); + end + v = sqrt( sum(x.^2,2) ); + s = 1 - 1 ./ max( v ./ ( t .* q ), 1 ); + x = bsxfun( @times, x, s ); % Apr 24 2013. Should be faster +end + +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_l1linf.m b/prox_l1linf.m new file mode 100644 index 0000000..8358b01 --- /dev/null +++ b/prox_l1linf.m @@ -0,0 +1,107 @@ +function op = prox_l1linf( q ) + +%PROX_L1LINF L1-LInf block norm: sum of L2 norms of rows. +% OP = PROX_L1LINF( q ) implements the nonsmooth function +% OP(X) = q * sum_{i=1:m} norm(X(i,:),Inf) +% where X is a m x n matrix. If n = 1, this is equivalent +% to PROX_L1 +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% then it must be positive and real. +% If Q is a vector, it must be m x 1, and in this case, +% the weighted norm OP(X) = sum_{i} Q(i)*norm(X(i,:),Inf) +% is calculated. + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || any(q <= 0), + error( 'Argument must be positive.' ); +end +op = tfocs_prox( @(x)f(x,q), @(x,t)prox_f(x,t,q), 'vector' ); +end % end of main function + +function v = f(x,q) + if numel(q) ~= 1 && size(q,1) ~= size(x,1) + error('Weight must be a scalar or a column vector'); + end + v = sum( q.* max(abs(x),[],2) ); +end + +function x = prox_f(x,t,q) + if nargin < 2, + error( 'Not enough arguments.' ); + end + + [n,d] = size(x); + dim = 2; + + % Option 1: explicitly call prox_linf on the rows: +% for k= 1:n +% if isscalar(q), qk = q; +% else, qk = q(k); +% end +% x(k,:) = prox_linf_q( qk, x(k,:).', t ).'; +% end +% return; + + + +% Option 2: vectorize the call. By far, more efficient than option 1 + s = sort( abs(x), dim, 'descend' ); + cs = cumsum(s,dim); + s = [s(:,2:end), zeros(n,1)]; + + + ndx1 = zeros(n,1); + ndx2 = zeros(n,1); + + if isscalar(q), + tq = repmat( t*q, n, d ); + else + tq = repmat( t*q, 1, d ); + end + Z = cs - s*diag(1:d); + Z = ( Z >= tq ); + Z = Z.'; + + + % Not sure how to vectorize the find. + % One option is to use the [i,j] = find(...) form, + % but that's also extra work, since we can't just find the "first". + % So for now, making a for loop + for k = 1:n + ndxk = find( Z(:,k), 1 ); + if ~isempty(ndxk) + ndx1(k) = ndxk; + ndx2(k) = ndxk; + else + ndx1(k) = 1; % value doesn't matter + ndx2(k) = Inf; + end + end + indx_cs = sub2ind( [n,d], (1:n)', ndx1 ); + tau = (cs(indx_cs) - tq(:,1))./ndx2; + tau = repmat( tau, 1, d ); + tau_noZeros = tau; + tau_noZeros( ~x ) = 1; + x = x .* ( tau ./ max( abs(x), tau_noZeros ) ); + + +end + +% Some old code: +% function x = prox_linf_q( q, x, t ) +% +% s = sort( abs(nonzeros(x)), 'descend' ); +% cs = cumsum(s); +% ndx = find( cs - (1:numel(s))' .* [s(2:end);0] >= t * q, 1 ); +% if ~isempty( ndx ), +% tau = ( cs(ndx) - t * q ) / ndx; +% x = x .* ( tau ./ max( abs(x), tau ) ); +% else +% x = 0; +% end +% end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_l1pos.m b/prox_l1pos.m new file mode 100644 index 0000000..f1e3e1e --- /dev/null +++ b/prox_l1pos.m @@ -0,0 +1,38 @@ +function op = prox_l1pos( q ) +%PROX_L1POS L1 norm, restricted to x >= 0 +% OP = PROX_L1( q ) implements the nonsmooth function +% OP(X) = norm(q.*X,1) + indicator_{ X >= 0 } +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% then it must be a positive real scalar (or must be same size as X). + +% New in v1.0d + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || any( q < 0 ) || all(q==0) %|| numel( q ) ~= 1 + error( 'Argument must be positive.' ); +end + +op = tfocs_prox( @(x)f(x,q), @(x,t)prox_f(x,t,q), 'vector'); +end + +function v = f(x,q) + if any( x(:) < 0 ) + v = Inf; + elseif isscalar(q) + v = q*sum( x(:) ); + else + v = sum( q(:).*x(:) ); + end +end + +% The proximity operator is a simplified version of shrinkage: +function x = prox_f(x,t,q) + x = max( 0, x - t*q ); +end + + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_l2.m b/prox_l2.m new file mode 100644 index 0000000..b2412c5 --- /dev/null +++ b/prox_l2.m @@ -0,0 +1,96 @@ +function op = prox_l2( q ) + +%PROX_L2 L2 norm. +% OP = PROX_L2( q ) implements the nonsmooth function +% OP(X) = q * norm(X,'fro'). +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% then it must be a positive real scalar. +% If Q is a vector or matrix of the same size and dimensions as X, +% then this uses an experimental code to compute the proximity operator +% of OP(x) = norm( q.*X, 'fro' ) +% In the limit q --> 0, this function acts like prox_0 (aka proj_Rn) +% Dual: proj_l2.m +% See also proj_l2, prox_0, proj_Rn + +% Feb '11, allowing for q to be a vector +% This is complicated, so not for certain +% A safer method is to use a linear operator to scale the variables + +if nargin == 0, + q = 1; +% elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, +elseif ~isnumeric( q ) || ~isreal( q ) %|| any( q < 0 ) || all(q==0) + error( 'Argument must be positive.' ); +end +if isscalar(q) + if any( q <= 0 ) + error('Scaling argument must be positive, real scalar. If q=0, use prox_0 instead'); + end + op = @(varargin)prox_l2_q( q, varargin{:} ); +else + if all(q==0), error('Argument must be nonzero'); end + warning('TFOCS:experimental','Using experimental feature of TFOCS'); + op = @(varargin)prox_l2_vector( q, varargin{:} ); +end + +function [ v, x ] = prox_l2_q( q, x, t ) +if nargin < 2, + error( 'Not enough arguments.' ); +end +v = sqrt( tfocs_normsq( x ) ); +if nargin == 3, + s = 1 - 1 ./ max( v / ( t * q ), 1 ); + + x = x * s; + v = v * s; % version A +elseif nargout == 2, + error( 'This function is not differentiable.' ); +end +v = q * v; % version A + + +% --------- experimental code ----------------------- +function [ v, x ] = prox_l2_vector( q, x, t ) +if nargin < 2, + error( 'Not enough arguments.' ); +end +v = sqrt( tfocs_normsq( q.*x ) ); % version B +if nargin == 3, +%{ + we need to solve for a scalar variable s = ||q.*x|| + (where x is the unknown solution) + + we have a fixed point equation: + s = f(s) := norm( q.*x_k ) where x_k = x_0/( 1 + t*q/s ) + + to solve this, we'll use Matlab's "fzero" to find the zero + of the function F(s) = f(s) - s + + Clearly, we need s >= 0, since it is ||q.*x|| + + If q is a scalar, we can solve explicitly: s = q*(norm(x0) - t) + +%} + + xk = @(s) x./( 1 + t*(q.^2)/s ); + f = @(s) norm( q.*xk(s) ); +% F = @(s) f(s) - s; + tq2 = t*(q.^2); + F = @(s) norm( (q.*x)./( 1 + tq2/s ) ) - s; + [s,sVal] = fzero( F, 1); + if abs( sVal ) > 1e-4 + error('cannot find a zero'); + end + if s <= 0 + x = 0*x; + else + x = xk(s); + end + v = sqrt( tfocs_normsq( q.*x ) ); % version B +elseif nargout == 2, + error( 'This function is not differentiable.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_linf.m b/prox_linf.m new file mode 100644 index 0000000..24e2fcc --- /dev/null +++ b/prox_linf.m @@ -0,0 +1,42 @@ +function op = prox_linf( q ) + +%PROX_LINF L-infinity norm. +% OP = PROX_LINF( q ) implements the nonsmooth function +% OP(X) = q * norm( X(:), Inf ). +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% then it must be a positive real scalar. +% Dual: proj_l1.m +% See also proj_l1 + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument must be positive.' ); +end +op = @(varargin)prox_linf_q( q, varargin{:} ); + +function [ v, x ] = prox_linf_q( q, x, t ) +if nargin < 2, + error( 'Not enough arguments.' ); +end +tau = norm( x(:), Inf ); +if nargin == 3, + s = sort( abs(nonzeros(x)), 'descend' ); % makes one big vector + + cs = cumsum(s); + ndx = find( cs - (1:numel(s))' .* [s(2:end);0] >= t * q, 1 ); + if ~isempty( ndx ), + tau = ( cs(ndx) - t * q ) / ndx; + x = x .* ( tau ./ max( abs(x), tau ) ); + else + x(:) = 0; % adding Oct 21 + tau = 0; + end +elseif nargout == 2, + error( 'This function is not differentiable.' ); +end +v = q * tau; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_max.m b/prox_max.m new file mode 100644 index 0000000..a3b7826 --- /dev/null +++ b/prox_max.m @@ -0,0 +1,53 @@ +function op = prox_max( q ) + +%PROX_MAX Entry-wise maximum element. +% OP = PROX_MAX( q ) implements the nonsmooth function +% OP(X) = q * max( X(:) ). +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% then it must be a positive real scalar. +% Dual: proj_simplex.m (at least if X is a vector) +% See also proj_simplex, prox_linf, prox_linf + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument must be positive.' ); +end +op = @(varargin)prox_lmax_q( q, varargin{:} ); + +function [ v, x ] = prox_lmax_q( q, x, t ) +if nargin < 2, + error( 'Not enough arguments.' ); +end +% We have two options when input x is a matrix: +% Does the user want to treat it as x(:), +% Or does the user want to treat each column separately? +% Most other functions (e.g. l1, linf) treat it as x(:) +% so that will be the default. However, we leave it +% as a hard-coded option so that the user can change it +% if they want. +% (Note: the dual function, proj_simplex, vectorizes it) +VECTORIZE = true; +% right now, we do not have the non-vectorized version implemented. + +tau = max( x(:) ); +if nargin == 3, + s = sort( nonzeros(x), 'descend' ); +% s = sort( x, 'descend' ); % 'nonzeros' does a x(:) operation + cs = cumsum(s); + ndx = find( cs - (1:numel(s))' .* [s(2:end);0] >= t * q, 1 ); + if ~isempty( ndx ), + tau = ( cs(ndx) - t * q ) / ndx; + x = x .* ( tau ./ max( x, tau ) ); + else + x(:) = 0; % adding Oct 21 + tau = 0; + end +elseif nargout == 2, + error( 'This function is not differentiable.' ); +end +v = q * tau; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_maxEig.m b/prox_maxEig.m new file mode 100644 index 0000000..b4d59d1 --- /dev/null +++ b/prox_maxEig.m @@ -0,0 +1,62 @@ +function op = prox_maxEig( q ) +%PROX_MAXEIG Maximum eigenvalue of a real symmetric matrix +% OP = PROX_MAXEIG( q ) implements the nonsmooth function +% OP(X) = q * max(eig(X)). +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% it must be a real scalar. +% +% Note: if X is positive semi-definite, then max(eig(X)) == norm(eig(X),Inf) +% and hence prox_maxEig(q) is equivalent to prox_spectral(q,'eig') +% +% This implementation uses a naive approach that does not exploit any +% a priori knowledge that X is low rank or sparse. Future +% implementations of TFOCS will be able to handle low-rank matrices +% more effectively. +% +% Dual: proj_psdUTrace.m +% See also proj_psdUTrace, prox_max, prox_spectral, proj_nuclear + +% Note: it would be possible to use eigs for sparse matrices, +% but that requires a little bit more work than it did for proj_spectral. + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 %|| q <= 0, + error( 'Argument must be a real scalar.' ); +end +op = @(varargin)prox_spectral_eig_q( q, varargin{:} ); + +function [ v, X ] = prox_spectral_eig_q( q, X, t ) +% Assumes X is square and symmetric +% Therefore, all singular values are just absolute values +% of the eigenvalues. +if nargin < 2, + error( 'Not enough arguments.' ); +end +if size(X,1) ~= size(X,2) + error('prox_spectral: variable must be a square matrix'); +end +if norm( X - X', 'fro' ) > 1e-10*norm(X,'fro') + error('Input must be Hermitian'); +end +X = (X+X')/2; % Matlab will make it exactly symmetric + +if nargin == 3 && t > 0, + [V,S] = eig(full(X)); +% op = prox_linf(q); % for prox_spectral + op = prox_max(q); + s = diag(S); + [dummy,s] = op(s,t); + tau = max(s); + X = V*diag(s)*V'; +else + if nargout == 2 + error( 'This function is not differentiable.' ); + end + tau = max(eig(X)); +end +v = q * tau; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_nuclear.m b/prox_nuclear.m new file mode 100644 index 0000000..4ff1adf --- /dev/null +++ b/prox_nuclear.m @@ -0,0 +1,140 @@ +function op = prox_nuclear( q, LARGESCALE ) + +%PROX_NUCLEAR Nuclear norm. +% OP = PROX_NUCLEAR( q ) implements the nonsmooth function +% OP(X) = q * sum(svd(X)). +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% it must be a positive real scalar. +% +% OP = PROX_NUCLEAR( q, LARGESCALE ) +% uses a Lanczos-based SVD if LARGESCALE == true, +% otherwise it uses a dense matrix SVD +% +% CALLS = PROX_NUCLEAR( 'reset' ) +% resets the internal counter and returns the number of function +% calls +% +% This implementation uses a naive approach that does not exploit any +% a priori knowledge that X and G are low rank or sparse. Future +% implementations of TFOCS will be able to handle low-rank matrices +% more effectively. +% Dual: proj_spectral.m +% See also prox_trace.m and proj_spectral.m + +if nargin == 1 && strcmpi(q,'reset') + op = prox_nuclear_impl; + return; +end + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument must be positive.' ); +end +if nargin < 2, LARGESCALE = []; end + +% clear the persistent values: +prox_nuclear_impl(); + +op = @(varargin)prox_nuclear_impl( q, LARGESCALE, varargin{:} ); + +end % end of main function + +function [ v, X ] = prox_nuclear_impl( q, LARGESCALE, X, t ) +persistent oldRank +persistent nCalls +if nargin == 0, oldRank = []; v = nCalls; nCalls = []; return; end +if isempty(nCalls), nCalls = 0; end + +ND = (size(X,2) == 1); +% ND = ~ismatrix(X); +if ND, % X is a vector, not a matrix, so reshape it + sx = size(X); + X = reshape( X, prod(sx(1:end-1)), sx(end) ); +end + +if nargin > 3 && t > 0, + + if ~isempty(LARGESCALE) % inherited from parent + largescale = LARGESCALE; + else + largescale = ( numel(X) > 100^2 ) && issparse(X); + end + tau = q*t; + nCalls = nCalls + 1; + +% fprintf('ranks: '); + if ~largescale + [U,S,V] = svd( full(X), 'econ' ); + else + % Guess which singular value will have value near tau: + [M,N] = size(X); + if isempty(oldRank), K = 10; + else, K = oldRank + 2; + end + + ok = false; + opts = []; + opts.tol = 1e-10; % the default in svds + opt = []; + opt.eta = eps; % makes compute_int slow +% opt.eta = 0; % makes reorth slow + opt.delta = 10*opt.eta; + while ~ok + K = min( [K,M,N] ); + if exist('lansvd','file') + [U,S,V] = lansvd(X,K,'L',opt ); + else + [U,S,V] = svds(X,K,'L',opts); + end + ok = (min(diag(S)) < tau) || ( K == min(M,N) ); +% fprintf('%d ',K ); + if ok, break; end +% K = K + 5; + K = 2*K; + if K > 10 + opts.tol = 1e-6; + end + if K > 40 + opts.tol = 1e-4; + end + if K > 100 + opts.tol = 1e-1; + end + if K > min(M,N)/2 +% disp('Computing explicit SVD'); + [U,S,V] = svd( full(X), 'econ' ); + ok = true; + end + end + oldRank = length(find(diag(S) > tau)); + end + s = diag(S) - tau; + tt = s > 0; + s = s(tt,:); + +% fprintf('\n')'; +% fprintf('rank is %d\n', length(tt) ); + + % Check to make sure this doesn't break existing code... + if isempty(s), +% X(:) = 0; % this line breaks the packSVD version + X = tfocs_zeros(X); + else + X = U(:,tt) * bsxfun( @times, s, V(:,tt)' ); + end +else + s = svd(full(X)); % could be expensive! +end + +v = q * sum(s); +if ND, + X = reshape( X, sx ); +end + +end + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_scale.m b/prox_scale.m new file mode 100644 index 0000000..351aed3 --- /dev/null +++ b/prox_scale.m @@ -0,0 +1,54 @@ +function op = prox_scale( proxF, scale ) + +%PROX_SCALE Scaling a proximity/projection function. +% PSCALE = PROX_SCALE( PROXF, s ) is the proximity function formed +% by multiplying the input by the real value SCALE, then calling the +% function PROXF. In other words, +% PSCALE( y ) = PROXF( s * y ). +% In three-argument form, [v,x]=PSCALE(y,t) finds the minimizer of +% PROXF( s * x ) + (0.5/t)*||x-y||_2^2 +% +% Why scale the input, and not the output? When the proximity function +% is a norm, the two choices are equivalent when SCALE >= 0, since +% ||s*x|| = abs(s)*||x||. However, for *indicator* functions, scaling +% the output is useless. Scaling the input grows or shrinks the domain +% of PSCALE by choosing SCALE<1 or SCALE>1, respectively. +% +% Note that when constructing our equivalence we assume a particular +% form for the proximity minimization. More generally, PROJ_SCALE will +% preserve equivalence if 0.5*||x-y||^2 is replaced with D(x-y), as +% long as D() satisfies the following homogeneity property: +% D(s*z) = s^2*D(z) +% If this is not the case, you will not be able to use the TFOCS +% internal scaling with this function. + +if ~isa( proxF, 'function_handle' ), + error( 'The first argument must be a function handle.' ); +elseif ~isa( scale, 'double' ) || ~isreal( scale ) || numel( scale ) ~= 1 || scale == 0, + error( 'The second argument must be a nonzero real scalar.' ); +elseif scale == 1, + op = proxF; +else + op = @(varargin)prox_scale_impl( proxF, scale, varargin{:} ); +end + +function varargout = prox_scale_impl( proxF, s, y, t ) +no = max(nargout,1); +if nargin < 4 || t == 0, + [ varargout{1:no} ] = proxF( s * y ); +else + % For three-input mode, we wish to simulate + % x = argmin_x projectorF(s*x) + (0.5/t)*||x-y||^2 + % Setting z = s * x, we have + % z = argmin_z projectorF(z) + (0.5/(s^2*t))*||z-s*y||^2 + % Therefore, we have + % [v,z] = projectorF( s * y, t * s^2 ); x = z / s; + [ varargout{1:no} ] = proxF( s * y, s^2 * t ); +end +if no > 1, + varargout{2} = varargout{2} / s; +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_spectral.m b/prox_spectral.m new file mode 100644 index 0000000..e51520f --- /dev/null +++ b/prox_spectral.m @@ -0,0 +1,105 @@ +function op = prox_spectral( q, SYM_FLAG ) +%PROX_SPECTRAL Spectral norm, i.e. max singular value. +% OP = PROX_SPECTRAL( q ) implements the nonsmooth function +% OP(X) = q * max(svd(X)). +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% it must be a positive real scalar. +% +% OP = PROX_SPECTRAL( q , 'sym' ) or OP = PROX_SPECTRAL( q, 'eig' ) +% will instruct the code that the matrix is Hermitian and +% therefore the relations between singular- and eigen-values +% are well known, and the code will use the more efficient +% eigenvalue decomposition (instead of the SVD). +% +% This implementation uses a naive approach that does not exploit any +% a priori knowledge that X is low rank or sparse. Future +% implementations of TFOCS will be able to handle low-rank matrices +% more effectively. +% +% Dual: proj_nuclear.m +% See also proj_nuclear, prox_linf, prox_maxEig + +% Note: it would be possible to use eigs for sparse matrices, +% but that requires a little bit more work than it did for proj_spectral. + + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument must be positive.' ); +end +if nargin >= 2 && ( ... + ~isempty(strfind(lower(SYM_FLAG),'eig')) || ... + ~isempty(strfind(lower(SYM_FLAG),'sym')) ) + op = @(varargin)prox_spectral_eig_q( q, varargin{:} ); +else + op = @(varargin)prox_spectral_impl( q, varargin{:} ); +end + +function [ v, X ] = prox_spectral_impl( q, X, t ) +if nargin < 2, + error( 'Not enough arguments.' ); +end + +if nargin == 3 && t > 0, + [U,S,V] = svd( full(X), 'econ' ); + s = diag(S); + tau = s(1); + + cs = cumsum(s); + ndx = find( cs - (1:numel(s))' .* [s(2:end);0] >= t * q, 1 ); + if ~isempty( ndx ), + tau = ( cs(ndx) - t * q ) / ndx; + s = s .* ( tau ./ max( abs(s), tau ) ); + end + X = U*diag(s)*V'; +else + if nargout == 2 + error( 'This function is not differentiable.' ); + end + if issparse(X) + tau = normest(X); + else + tau = norm(X); + end +end +v = q * tau; + +function [ v, X ] = prox_spectral_eig_q( q, X, t ) +% Assumes X is square and symmetric +% Therefore, all singular values are just absolute values +% of the eigenvalues. +if nargin < 2, + error( 'Not enough arguments.' ); +end +if size(X,1) ~= size(X,2) + error('prox_spectral: variable must be a square matrix'); +end +if norm( X - X', 'fro' ) > 1e-10*norm(X,'fro') + error('Input must be Hermitian'); +end +X = (X+X')/2; % Matlab will make it exactly symmetric + +if nargin == 3 && t > 0, + [V,S] = eig(full(X)); + op = prox_linf(q); + s = diag(S); + [dummy,s] = op(s,t); +% tau = max(s); % for SVD, s >= 0 so we can do max. Not so for eig. + tau = norm(s,Inf); + X = V*diag(s)*V'; +else + if nargout == 2 + error( 'This function is not differentiable.' ); + end + if issparse(X) + tau = normest(X); + else + tau = norm(X); + end +end +v = q * tau; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/prox_trace.m b/prox_trace.m new file mode 100644 index 0000000..b18f37c --- /dev/null +++ b/prox_trace.m @@ -0,0 +1,139 @@ +function op = prox_trace( q, LARGESCALE, isReal ) + +%PROX_TRACE Nuclear norm, for positive semidefinite matrices. Equivalent to trace. +% OP = PROX_TRACE( q ) implements the nonsmooth function +% OP(X) = q * sum(svd(X)) = q*tr(X) ( X >= 0 assumed ) +% Q is optional; if omitted, Q=1 is assumed. But if Q is supplied, +% it must be a positive real scalar. +% This function is a combination of the proximity function of the trace +% and projection onto the set of symmetric/Hermitian matrices. +% +% OP = PROX_TRACE( q, LARGESCALE ) +% uses a Lanczos-based Eigenvalue decomposition if LARGESCALE == true, +% otherwise it uses a dense matrix Eigenvalue decomposition +% +% OP = PROX_TRACE( q, LARGESCALE, isReal ) +% also projects onto the set of real matrices if isReal=true. +% +% CALLS = PROX_TRACE( 'reset' ) +% resets the internal counter and returns the number of function +% calls +% +% This implementation uses a naive approach that does not exploit any +% a priori knowledge that X and G may be low rank (plus sparse). Future +% implementations of TFOCS will be able to handle low-rank matrices +% more effectively. +% Dual: proj_spectral(q,'symm') +% See also proj_spectral, prox_nuclear + +if nargin == 1 && strcmpi(q,'reset') + op = prox_trace_impl; + return; +end + +if nargin == 0, + q = 1; +elseif ~isnumeric( q ) || ~isreal( q ) || numel( q ) ~= 1 || q <= 0, + error( 'Argument must be positive.' ); +end +if nargin < 2, LARGESCALE = []; end +if nargin < 3 || isempty(isReal), isReal = false; end + +% clear the persistent values: +prox_trace_impl(); + +op = @(varargin)prox_trace_impl( q, LARGESCALE, isReal, varargin{:} ); +end + +function [ v, X ] = prox_trace_impl( q,LARGESCALE, isReal, X, t ) +persistent oldRank +persistent nCalls +persistent V +if nargin == 0, oldRank = []; v = nCalls; nCalls = []; V=[]; return; end +if isempty(nCalls), nCalls = 0; end + +if nargin >= 5 && t > 0, + + if ~isempty(LARGESCALE) + largescale = LARGESCALE; + else + largescale = ( numel(X) > 100^2 ) && issparse(X); + end + tau = q*t; + nCalls = nCalls + 1; + + if ~largescale + if isReal + [V,D] = eig(real(full((X+X')/2))); + else + [V,D] = eig(full((X+X')/2)); + end + else + + % Guess which eigenvalue value will have value near tau: + [M,N] = size(X); + X = (X+X')/2; + if isReal, X = real(X); end + if isempty(oldRank), K = 10; + else, K = oldRank + 2; + end + + ok = false; + opts = []; + opts.tol = 1e-10; + if isreal(X) + opts.issym = true; + SIGMA = 'LA'; + else + SIGMA = 'LR'; + end + % SIMGA = 'LM' (bug) prior to March 18 2012 + while ~ok + K = min( [K,M,N] ); + + if K > min(M,N)/2 + [V,D] = eig(full((X+X')/2)); + ok = true; + break; + end + + [V,D] = eigs( X, K, SIGMA, opts ); + ok = (min(diag(D)) < tau) || ( K == min(M,N) ); + if ok, break; end + K = 2*K; + if K > 10 + opts.tol = 1e-6; + end + if K > 40 + opts.tol = 1e-4; + end + if K > 100 + opts.tol = 1e-3; + end + + end + oldRank = length(find(diag(D) > tau)); + end + s = diag(D) - tau; + tt = s > 0; + s = s(tt,:); + + if isempty(s), + X = tfocs_zeros(X); + else + X = V(:,tt) * bsxfun( @times, s, V(:,tt)' ); + % And force it to be symmetric + X = (X+X')/2; + end + v = q * sum(s); + +else + v = q* trace( X + X' )/2; +end + +end + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/smooth_constant.m b/smooth_constant.m new file mode 100644 index 0000000..038249a --- /dev/null +++ b/smooth_constant.m @@ -0,0 +1,29 @@ +function op = smooth_constant( d ) + +%SMOOTH_CONSTANT Constant function generation. +% FUNC = SMOOTH_CONSTANT( D ) returns a function handle that provides +% a TFOCS-compatible implementation of the constant function F(X) = D. +% D must be a real scalar. The function can be used in both a smooth +% and a nonsmooth context. + +error(nargchk(1,1,nargin)); +if ~isa( d, 'double' ) || ~isreal( d ) || numel( d ) ~= 1, + error( 'Argument must be a real scalar.' ); +end +op = @(varargin)smooth_constant_impl( d, varargin{:} ); + +function [ v, g ] = smooth_constant_impl( v, x, t ) +switch nargin, + case 2, + if nargin > 1, + g = 0 * x; + end + case 3, + g = x; + otherwise, + error( 'Not enough arguments.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/smooth_entropy.m b/smooth_entropy.m new file mode 100644 index 0000000..6f2d32e --- /dev/null +++ b/smooth_entropy.m @@ -0,0 +1,21 @@ +function op = smooth_entropy() +%SMOOTH_ENTROPY The entropy function -sum( x_i log(x_i) ) +op = @smooth_entropy_impl; + +function [ v, g ] = smooth_entropy_impl( x ) +if any( x < 0 ), + v = -Inf; + if nargout > 1, + g = NaN * ones(size(x)); + end +else + logx = log(max(x,realmin)); + v = - tfocs_dot( x, logx ); + if nargout > 1, + g = - logx - 1; + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/smooth_handles.m b/smooth_handles.m new file mode 100644 index 0000000..e15ee7a --- /dev/null +++ b/smooth_handles.m @@ -0,0 +1,27 @@ +function op = smooth_handles( func, grad ) + +%SMOOTH_HANDLES Smooth function from separate f/g handles. +% OP = SMOOTH_HANDLES( func, grad ) constructs a TFOCS-compatible +% smooth function from separate handles for computing the function +% value and gradient. +% +% See also private/tfocs_smooth + +op = @(varargin)smooth_handles_impl( func, grad, varargin{:} ); + +function [ f, g ] = smooth_handles_impl( fop, gop, x, t ) +switch nargin, + case 3, + f = fop( x ); + if nargout > 1, + g = gop( x ); + end + case 4, + error( 'This function does not support proximity minimizaztion.' ); + case { 0, 1, 2 }, + error( 'Not enough input arguments.' ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/smooth_huber.m b/smooth_huber.m new file mode 100644 index 0000000..63f6f07 --- /dev/null +++ b/smooth_huber.m @@ -0,0 +1,47 @@ +function op = smooth_huber(tau ) + +%SMOOTH_HUBER Huber function generation. +% FUNC = SMOOTH_QUAD( TAU ) returns a function handle that implements +% +% FUNC(X) = 0.5 *( x.^2 )/tau if |x| <= tau +% = |x| - tau/2 if |x| > tau +% +% All arguments are optional; the default value is tau = 1. +% The Huber function has continuous gradient and is convex. +% +% The function acts component-wise. TAU may be either a scalar +% or a vector/matrix of the same size as X +% +% Does not support nonsmooth usage yet + +% Does not yet fully support tfocs_tuples + +if nargin == 0, + tau = 1; +end +% op = @(varargin) smooth_huber_impl(tau, varargin{:} ); % old method + +op = tfocs_smooth( @smooth_huber_impl ); + + +% function [ v, g ] = smooth_huber_impl(tau, x, t ) % old method +function [ v, g ] = smooth_huber_impl(x) + if nargin == 3, + error( 'Proximity minimization not supported by this function.' ); + end + if ~isscalar(tau) && ~size(tau) == size(x) + error('smooth_huber: tau must be a scalar or the same size as the variable'); + end + smallSet = ( abs(x) <= tau ); + v = smallSet.*( 0.5*(x.^2)./tau ) + (~smallSet).*( abs(x) - tau/2) ; + if nargout > 1 + g = sign(x).*min( 1, abs(x)./tau ); + end +end % new method + +end % new method + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/smooth_linear.m b/smooth_linear.m new file mode 100644 index 0000000..db272e9 --- /dev/null +++ b/smooth_linear.m @@ -0,0 +1,38 @@ +function op = smooth_linear( c, d ) + +%SMOOTH_LINEAR Linear function generation. +% FUNC = SMOOTH_LINEAR( C, D ) returns a function handle that provides a +% TFOCS-compatible implementation of a linear function: if +% [F,G] = FUNC(X), +% then F = TFOCS_DOT(C,X)+D and G = C. D is optional; if omitted, then +% D == 0 is assumed. But if it is supplied, D must be a real scalar. +% If C == 0, then this function is equivalent to SMOOTH_CONSTANT( D ). +% This function can be used in both smooth and non-smooth contexts. + +error(nargchk(1,2,nargin)); +if nargin < 2, + d = 0; +elseif ~isa( d, 'double' ) || ~isreal( d ) || numel( d ) ~= 1, + error( 'Second argument must be a real scalar.' ); +end +if nnz(c), + op = @(varargin)smooth_linear_impl( c, d, varargin{:} ); +else + op = smooth_constant( d ); +end + +function [ v, g ] = smooth_linear_impl( c, d, x, t ) +switch nargin, + case 3, + g = c; + case 4, + x = x - t * c; + g = x; + otherwise, + error( 'Not enough arguments.' ); +end +v = tfocs_dot( c, x ) + d; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/smooth_logLLogistic.m b/smooth_logLLogistic.m new file mode 100644 index 0000000..c8dd0b9 --- /dev/null +++ b/smooth_logLLogistic.m @@ -0,0 +1,41 @@ +function op = smooth_logLLogistic(y) +% SMOOTH_LOGLLOGISTIC Log-likelihood function of a logistic: sum_i( y_i mu_i - log( 1+exp(mu_i) ) ) +% OP = SMOOTH_LOGLLOGISTIC( Y ) +% returns a function that computes the log-likelihood function +% in a standard logistic regression model with independent entries. There +% are two classes y_i = 0 and y_i = 1 with +% +% prob(y_i = 1) = exp(mu_i)/(1 + exp(mu_i) +% +% so that the log-likelihood is given by +% +% log-likelihood(mu) = sum_i ( y_i mu_i - log(1+ exp(mu_i)) ) +% +% where mu is the parameter of the distribution (this is unknown, +% so it is the variable), and Y is a vector of observations. + +error(nargchk(1,1,nargin)); +op = tfocs_smooth( @smooth_logLlogistic_impl); + +function [ v, g ] = smooth_logLlogistic_impl( mu ) + + if length(mu) == 1, + mu = mu * ones(size(y)); + elseif size(mu) ~= size(y), + error('Parameters and data must be of the same size'), + end + + aux = 1 + exp(-abs(mu)); + v = tfocs_dot(y-1,mu.*(mu > 0)) ... + + tfocs_dot(y,mu.*(mu < 0)) ... + - tfocs_dot(ones(size(y)), log(aux)); + if nargout > 1, + g = y - ((mu > 0) + (mu <= 0).*exp(mu))./aux; + end +end + +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/smooth_logLPoisson.m b/smooth_logLPoisson.m new file mode 100644 index 0000000..9c14ea3 --- /dev/null +++ b/smooth_logLPoisson.m @@ -0,0 +1,43 @@ +function op = smooth_logLPoisson(x) +% SMOOTH_LOGLPOISSON Log-likelihood of a Poisson: sum_i (-lambda_i + x_i * log( lambda_i) ) +% OP = SMOOTH_LOGLPOISSON( X ) +% returns a function that computes the log-likelihood function +% of independent Poisson random variables with parameters lambda_i: +% +% log-likelihood(lambda) = sum_i ( -lambda_i + x_i * log( lambda_i) ) +% +% where LAMBDA is the parameter of the distribution (this is unknown, +% so it is the variable), and X is a vector of observations. +% +% Note: the constant term in the log-likelihood is omitted. + +error(nargchk(1,1,nargin)); +op = tfocs_smooth( @smooth_llPoisson_impl ); + +function [ v, g ] = smooth_llPoisson_impl( lambda ) + + if length(lambda) == 1, + lambda = lambda * ones(size(x)); + elseif size(lambda) ~= size(x), + error('Parameters and data must be of the same size'), + end + + if any( lambda < 0 ), + v = -Inf; + if nargout > 1, + g = NaN * ones(size(x)); + end + else + loglambda = log(max(lambda,realmin)); + v = - tfocs_dot(lambda, ones(size(x))) + tfocs_dot( x, loglambda); + if nargout > 1, + g = -1 + x./lambda; + end + end +end + +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/smooth_logdet.m b/smooth_logdet.m new file mode 100644 index 0000000..0f7b9c3 --- /dev/null +++ b/smooth_logdet.m @@ -0,0 +1,127 @@ +function op = smooth_logdet(q,C) +% SMOOTH_LOGDET The -log( det( X ) ) function. +% (Note the minus sign) +% FUNC = SMOOTH_LOGDET( q ) returns a function handle that +% provides a TFOCS-compatible implementation of the funciton +% -q*log( det( X ) ) +% +% FUNC = SMOOTH_LOGDET( q, C ) represents +% -q*log( det( X ) ) + < C, X >, where C is symmetric/Hermitian +% +% X must be symmetric/Hermitian and positive definite, +% and q must be a positive real number (if not provided, +% the default value is q = 1). +% +% N.B. it is the user's responsibility to ensure +% that X is Hermitian and pos. def., since +% automatically checking is expensive. +% +% This function is differentiable, but the gradient +% is not Lipschitz on the domain X > 0 +% +% This function does support proximity operations, and so +% it may be used as a nonsmooth function. +% However, the input must be symmetric positive definite +% (and if C is used, then it must also be > tC ) + +% SRB: have not yet tested this. +% SRB: I think we CAN compute the proximity operator to logdet. +% Will implement this in prox_logdet +if nargin < 1, q = 1; end +if nargin < 2, C = []; end +if ~isreal(q) || q <= 0 + error('First argument must be real and positive'); +end + +%op = @smooth_logdet_impl; +if isempty(C) + op = @(varargin)smooth_logdet_impl( q, varargin{:} ); +else + op = @(varargin)smooth_logdet_impl_C( q, C, varargin{:} ); +end + +function [ v, g ] = smooth_logdet_impl( q, x, t ) +if size(x,1) ~= size(x,2) + error('smooth_logdet: input must be a square matrix'); +end +switch nargin + case 2 + % the function is being used in a "smooth" fashion + %v = -log(det(x)); + v = -2*q*sum(log(diag(chol(x)))); % chol() takes half the time as det() + % and it is easier to avoid overflow errors + % since we sum the logs. + % Also, chol() will warn if not pos. def. + if nargout > 1 + g = -q*inv(x); + % it would be nice to make g a function handle + % that calculates g(y) = -x\y + end + + + case 3 + % the function is being used in a "nonsmooth" fashion + % i.e. return g = argmin_g -q*log(det(g)) + 1/(2t)||g-x||^2 + [V,D] = eig(x); + d = diag(D); + if any(d<=0), + v = Inf; + g = nan(size(x)); + return; +% error('log_det requires a positive definite point'); + end + l = ( d + sqrt( d.^2 + 4*t*q ) )/2; + g = V*diag(l)*V'; + v = -q*sum(log(l)); + otherwise + error('Wrong number of arguments'); +end + + +function [ v, g ] = smooth_logdet_impl_C( q, C, x, t ) +if size(x,1) ~= size(x,2) + error('smooth_logdet: input must be a square matrix'); +end +if size(C,1) ~= size(C,2) + error('smooth_logdet: input must be a square matrix'); +end +switch nargin + case 3 + % the function is being used in a "smooth" fashion + %v = -log(det(x)); + v = -2*q*sum(log(diag(chol(x)))); % chol() takes half the time as det() + % and it is easier to avoid overflow errors + % since we sum the logs. + % Also, chol() will warn if not pos. def. + v = v + tfocs_dot( C, x ); + if nargout > 1 + g = -q*inv(x) + C; + % it would be nice to make g a function handle + % that calculates g(y) = -x\y + end + + + case 4 + % the function is being used in a "nonsmooth" fashion + % i.e. return g = argmin_g -q*log(det(g)) + 1/(2t)||g-x||^2 + x = x - t*C; % hope that this remains pos. def.! + [V,D] = eig(x); + d = diag(D); + if any(d<=0), + v = Inf; + g = nan(size(x)); + return; +% error('log_det requires a positive definite point'); + end + l = ( d + sqrt( d.^2 + 4*t*q ) )/2; + g = V*diag(l)*V'; + v = -q*sum(log(l)); + v = v + tfocs_dot( C, g ); + otherwise + error('Wrong number of arguments'); +end + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/smooth_logsumexp.m b/smooth_logsumexp.m new file mode 100644 index 0000000..ed2d716 --- /dev/null +++ b/smooth_logsumexp.m @@ -0,0 +1,21 @@ +function op = smooth_logsumexp() +% SMOOTH_LOGSUMEXP The function log(sum(exp(x))) +% returns a smooth function to calculate +% log( sum( exp(x) ) ) +% +% For a fancier version (with offsets), +% see also smooth_LogLLogistic.m + +op = @smooth_logsumexp_impl; + +function [ v, g ] = smooth_logsumexp_impl( x ) +expx = exp(x); +sum_expx = sum(expx(:)); +v = log(sum_expx); +if nargout > 1, + g = expx ./ sum_expx; +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/smooth_quad.m b/smooth_quad.m new file mode 100644 index 0000000..4ccd935 --- /dev/null +++ b/smooth_quad.m @@ -0,0 +1,152 @@ +function op = smooth_quad( P, q, r, use_eig ) + +%SMOOTH_QUAD Quadratic function generation. +% FUNC = SMOOTH_QUAD( P, q, r ) returns a function handle that implements +% +% FUNC(X) = 0.5 * TFOCS_DOT( P * x, x ) + TFOCS_DOT( q, x ) + r. +% +% All arguments are optional; the default values are P=I, q=0, r=0. In +% particular, calling FUNC = SMOOTH_QUAD with no arguments yields +% +% FUNC(X) = 0.5 * TFOCS_NORMSQ( X ) = 0.5 * TFOCS_DOT( X, X ). +% +% If supplied, P must be a scalar, square matrix, or symmetric linear +% operator. Furthermore, it must be positive semidefinite (convex) or +% negative semidefinite (concave). TFOCS does not verify operator +% symmetry or definiteness; that is your responsibility. +% If P is a vector, then it assumed this is the diagonal part of P. +% Note: when P is diagonal, this function can compute a proximity +% operator. If P is zero, then smooth_linear should be called instead. +% When P is an explicit matrix, this can also act as a proimity operator +% but it may be slow, since it must invert (I+tP). True use_eig mode (see below) +% +% If P is a scaling matrix, like the identity or a multiple of the identity +% (say, P*x = 5*x), then specifying the scaling factor is sufficient (in +% this example, 5). If P is empty, then P=1 is assumed. +% +% FUNC = SMOOTH_QUAD( P, q, r, use_eig ) +% will perform a one-time (but expensive) eigenvalue decomposition +% of P if use_eig=true, which will speed up future iterations. In general, +% this may be significantly faster, especially if you run +% an algorithm for many iterations. +% This mode is only useful when P is a full matrix and when +% smooth_quad is used as a proximity operator; does not affect +% it when used as a smooth operator. New in v 1.3. +% +% See also smooth_linear.m + +if nargin == 0, + op = @smooth_quad_simple; + return +end +if isa( P, 'function_handle' ), + sz = P([],0); + if ~isequal( sz{1}, sz{2} ), + error( 'P must be square.' ); + end +elseif ~isnumeric( P ), + error( 'P must be a scalar, matrix, or linear operator.' ); +elseif isempty(P) + P = 1; +elseif ndims( P ) > 2 || (~isvector(P) && size( P, 1 ) ~= size( P, 2 ) ), + error( 'P must be a square matrix.' ); +end +if nargin < 2 || isempty( q ), + q = 0; +elseif numel(P) > 1 && numel(q) > 1 && length(q) ~= size(P,1), + error( 'Dimension mismatch between p and q.' ); +end +if nargin < 3 || isempty( r ), + r = 0; +elseif numel(r) > 1 || ~isreal( r ), + error( 'r must be a real scalar.' ); +end +if nargin < 4, use_eig = false; end + +if isnumeric( P ), + if isvector( P ) + if any(P) < 0 && any(P) < 0 + error(' P must be convex (minimization) or concave (maximization) but cannot be mixed'); + end + P = P(:); % make it a column vector + op = @(varargin)smooth_quad_diag_matrix( P, q, r, varargin{:} ); + else + P = 0.5 * ( P + P' ); + if use_eig + [V,DD] = eig(P); dd = diag(DD); + else + V = []; dd = []; + end + op = @(varargin)smooth_quad_matrix( P, q, r, V, dd, varargin{:} ); + end +else + op = @(varargin)smooth_quad_linop( P, q, r, varargin{:} ); +end + +function [ v, g ] = smooth_quad_matrix( P, q, r, V, dd, x, t ) +switch nargin + case 6, + g = P * x + q; + v = 0.5 * tfocs_dot( x, g + q ) + r; + case 7, + n = length(x); + + if ~isempty(V) && ~isempty(dd) + % Use the stored eigenvalue decomposition to improve speed + x = V*( (V'*(x-t*q))./(t*dd+ones(n,1)) ); + else + x = (eye(n) + t*P)\ (x-t*q); + end + g = x; % for this case, "g" is NOT the gradient + v = 0.5 * tfocs_dot( P*x + 2*q, x ) + r; +end +%if nargin == 5, + %error( 'Proximity minimization not supported by this function.' ); +%end +% Note: we don't support proximity minimization, but there is +% nothing that prevents it theoretically. You would need to know P^-1 +% (and ideally calculate it quickly). +% In particular, if P is diagonal, then it is easy. +%g = P * x + q; +%v = 0.5 * tfocs_dot( x, g + q ) + r; + + + +% Jan 10, 2011: this function isn't correct for case 5 +% function [ v, g ] = smooth_quad_diag_matrix( p, q, r, x, t ) +% switch nargin +% case 4, +% case 5, +% x = (1./(t*p+1)) .* x; +% end +% g = p .* x + q; +% v = 0.5 * tfocs_dot( x, g + q ) + r; +function [ v, g ] = smooth_quad_diag_matrix( p, q, r, x, t ) +switch nargin + case 4, + g = p .* x + q; + v = 0.5 * tfocs_dot( x, g + q ) + r; + case 5, + x = (1./(t*p+1)) .* (x-t*q); + g = x; % for this case, "g" is NOT the gradient + v = 0.5 * tfocs_dot( p.*x + 2*q, x ) + r; +end + +function [ v, g ] = smooth_quad_linop( P, q, r, x, t ) +if nargin == 5, + error( 'Proximity minimization not supported by this function.' ); +end +g = P( x, 1 ) + q; +v = 0.5 * tfocs_dot( x, g + q ) + r; + +function [ v, x ] = smooth_quad_simple( x, t ) +switch nargin, + case 1, + case 2, + x = (1/(t+1)) * x; +end +v = 0.5 * tfocs_normsq( x ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/solver_L1RLS.m b/solver_L1RLS.m new file mode 100644 index 0000000..8d77c86 --- /dev/null +++ b/solver_L1RLS.m @@ -0,0 +1,25 @@ +function [ x, odata, opts ] = solver_L1RLS( A, b, lambda, x0, opts ) +% SOLVER_L1RLS l1-regularized least squares problem, sometimes called the LASSO. +% [ x, odata, opts ] = solver_L1RLS( A, b, lambda, x0, opts ) +% Solves the l1-regularized least squares problem +% minimize (1/2)*norm( A * x - b )^2 + lambda * norm( x, 1 ) +% using the Auslender/Teboulle variant with restart. A must be a matrix +% or a linear operator, b must be a vector, and lambda must be a real +% positive scalar. The initial point x0 and option structure opts are +% both optional. +% +% Note: this formulation is sometimes referred to as "The Lasso" + +error(nargchk(3,5,nargin)); +if nargin < 4, x0 = []; end +if nargin < 5, opts = []; end +if ~isfield( opts, 'restart' ), + opts.restart = 100; +end + +[x,odata,opts] = tfocs( smooth_quad, { A, -b }, prox_l1( lambda ), x0, opts ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_LASSO.m b/solver_LASSO.m new file mode 100644 index 0000000..fd3785e --- /dev/null +++ b/solver_LASSO.m @@ -0,0 +1,27 @@ +function [ x, out, opts ] = solver_LASSO( A, b, tau, x0, opts ) +% SOLVER_LASSO Minimize residual subject to l1-norm constraints. +% [ x, out, opts ] = solver_LASSO( A, b, tau, x0, opts ) +% Solves the LASSO in the standard Tibshirani formulation, +% minimize (1/2)*norm(A*x-b)^2 +% s.t. norm(x,1) <= tau +% A must be a linear operator or matrix, and b must be a vector. The +% initial point x0 and the options structure opts are optional. +% +% Note: many people use "LASSO" to refer to the problem: +% minimize 1/2*norm(A*x-b)^2 + lambda*norm(x,1) +% In TFOCS, this version is implemented in "solver_L1RLS" +% (which stands for "L1 regularized Least Squares" ) + +% Supply default values +error(nargchk(3,5,nargin)); +if nargin < 4, x0 = []; end +if nargin < 5, opts = []; end +if ~isfield( opts, 'restart' ), opts.restart = 100; end + +% Extract the linear operators +[x,out,opts] = tfocs( smooth_quad, { A, -b }, proj_l1( tau ), x0, opts ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_OrderedLASSO.m b/solver_OrderedLASSO.m new file mode 100644 index 0000000..e300af0 --- /dev/null +++ b/solver_OrderedLASSO.m @@ -0,0 +1,29 @@ +function [ x, odata, opts ] = solver_OrderedLASSO( A, b, lambda, x0, opts ) +% SOLVER_ORDEREDLASSO l1-regularized least squares problem, sometimes called the LASSO, +% [ beta, odata, opts ] = solver_L1RLS( X, y, lambda, beta0, opts ) +% Solves the l1-regularized least squares problem, using the ordered l1 norm, +% minimize (1/2)*norm( A * x - b )^2 + norm( lasso.*sort(abs(x),'descend'), 1 ) +% using the Auslender/Teboulle variant with restart. X must be a matrix +% or a linear operator, y must be a vector, and lambda must be a real +% positive vector in decreasing order. +% The initial point beta0 and option structure opts are both optional. +% Reference: +% "Statistical Estimation and Testing via the Ordered l1 Norm" +% by M. Bogdan, E. van den Berg, W. Su, and E. J. Candès, 2013 +% http://www-stat.stanford.edu/~candes/OrderedL1/ +% +% See also solver_L1RLS.m, solver_LASSO.m, prox_Ol1.m + +error(nargchk(3,5,nargin)); +if nargin < 4, x0 = []; end +if nargin < 5, opts = []; end +if ~isfield( opts, 'restart' ), + opts.restart = 100; +end + +[x,odata,opts] = tfocs( smooth_quad, { A, -b }, prox_Ol1( lambda ), x0, opts ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_TraceLS.m b/solver_TraceLS.m new file mode 100644 index 0000000..6840c6d --- /dev/null +++ b/solver_TraceLS.m @@ -0,0 +1,48 @@ +function [ x, odata, opts ] = solver_TraceLS( A, b, lambda, x0, opts ) +% SOLVER_TRACELS Unconstrained form of trace-regularized least-squares problem. +% [ x, odata, opts ] = solver_TraceLS( A, b, lambda, x0, opts ) +% Solves the trace-regularized least squares problem +% minimize (1/2)*norm( A * X - b )^2 + lambda * trace( X ) +% with the constraint that X is positive semi-definite. +% A must be a matrix or a linear operator, b must be a vector, +% and lambda must be a real positive scalar. +% The initial point x0 and option structure opts are +% both optional. +% +% If "A" is a sparse matrix, and nnz(Z) = length(b), then it is assumed +% that the nonzero entries correspond to samples, and then the +% corresponding sampling operator is used. +% +% If opts.largescale = true, then uses an iterative method +% to compute the eigenvalue decomposition used with prox_trace. +% +% See also solver_L1RLS (aka The Lasso) + + +% Added Feb 7, 2011 +error(nargchk(3,5,nargin)); +if nargin < 4, x0 = []; end +if nargin < 5, opts = []; end +if ~isfield( opts, 'restart' ), + opts.restart = 100; +end +if issparse(A) && nnz(A) == length(b) + A = linop_subsample(A); +end +if isfield( opts,'largescale') + prx = prox_trace( lambda, opts.largescale ); + opts = rmfield(opts,'largescale'); +else + prx = prox_trace( lambda ); +end +% Note: the proximity operator of trace is really a combination +% of the proximity operator of trace and the indicator +% set of positive semi-definite matrices. +% So we do not need to *explicitly* include the positive semi-definite constraint. + +[x,odata,opts] = tfocs( smooth_quad, { A, -b }, prx, x0, opts ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_psdComp.m b/solver_psdComp.m new file mode 100644 index 0000000..bffd282 --- /dev/null +++ b/solver_psdComp.m @@ -0,0 +1,66 @@ +function [ x, out, opts ] = solver_psdComp( Xinc, opts ) +% SOLVER_PSDCOMP Matrix completion for PSD matrices. +% [ x, out, opts ] = solver_psdComp( Xinc, opts ) +% Solves the PSD matrix completion problem +% minimize (1/2)*norm(X(ij)-vv).^2 +% s.t. X p.s.d +% where ij is a vector of indices corresponding to the known elements. +% The nonzero values of Xinc are assumed to be the known values; that +% is, all zero values are considered unknowns. In order to specify a +% known zero value, replace it with a very small value; e.g., 1e-100. +% +% Since X must be symmetric, the ij entries should have symmetry. If they +% are not specified symmetrically, then choose opts.symmetrize = true +% to force them to become symmetric. This will change the objective +% function. +% +% Set opts.largescale=true to use eigs() instead if eig(). +% +% See also solver_psdCompConstrainedTrace + +% Supply default values +error(nargchk(1,2,nargin)); +if nargin < 2, opts = []; end +if ~isfield( opts, 'restart' ), + opts.restart = 50; +end +if isfield( opts, 'symmetrize' ), + symmetrize = opts.symmetrize; + opts = rmfield(opts,'symmetrize'); +else + symmetrize = false; +end +if isfield( opts, 'largescale' ), + largescale = opts.largescale; + opts = rmfield(opts,'largescale'); +else + largescale = false; +end +[n,m] = size(Xinc); +if n ~= m, error( 'Input must be square.' ); end + +if norm(Xinc-Xinc','fro') > 1e-8*norm(Xinc,'fro') && symmetrize + Xinc = symmetrizeSparseMatrix(Xinc); + disp('symmetrizing input'); +end +linop = linop_subsample(Xinc); +vv = full( Xinc( find(Xinc) ) ); + +[x,out,opts] = tfocs( smooth_quad, { linop, -vv }, proj_psd(largescale), Xinc, opts ); + + +function X = symmetrizeSparseMatrix( X ) + % Some entries were only specified once, others were + % double-specified, so we don't know if we should + % divide by 1 or 2 + ind1 = find( tril(X) ); + ind2 = find( tril(X',-1) ); + X = tril(X) + tril( X', -1); + ind = intersect(ind1,ind2); + X(ind) = X(ind)/2; + X = X + tril(X,-1)'; + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/solver_psdCompConstrainedTrace.m b/solver_psdCompConstrainedTrace.m new file mode 100644 index 0000000..4e09483 --- /dev/null +++ b/solver_psdCompConstrainedTrace.m @@ -0,0 +1,84 @@ +function [ x, out, opts ] = solver_psdCompConstrainedTrace( Xinc, tr, opts ) +% SOLVER_PSDCOMPCONSTRAINEDTRACE Matrix completion with constrained trace, for PSD matrices. +% [ x, out, opts ] = solver_psdCompConstrainedTrace( Xinc, tr, opts ) +% Solves the PSD matrix completion problem +% minimize (1/2)*norm(X(ij)-vv).^2 +% s.t. X p.s.d and trace(X) = tr +% where ij is a vector of indices corresponding to the known elements. +% The nonzero values of Xinc are assumed to be the known values; that +% is, all zero values are considered unknowns. In order to specify a +% known zero value, replace it with a very small value; e.g., 1e-100. +% +% Since X must be symmetric, the ij entries should have symmetry. If they +% are not specified symmetrically, then choose opts.symmetrize = true +% to force them to become symmetric. This will change the objective +% function. +% +% See also solver_psdComp + +% Supply default values +error(nargchk(1,3,nargin)); +if nargin < 3, opts = []; end +if ~isfield( opts, 'restart' ), + opts.restart = 50; +end +if isfield( opts, 'symmetrize' ), + symmetrize = opts.symmetrize; + opts = rmfield(opts,'symmetrize'); +else + symmetrize = false; +end +if nargin < 2 || isempty(tr) + tr = 1; +end + +[n,m] = size(Xinc); +if n ~= m, error( 'Input must be square.' ); end +% If not symmetric, then make it symmetric +if norm(Xinc-Xinc','fro') > 1e-8*norm(Xinc,'fro') && symmetrize + Xinc = symmetrizeSparseMatrix(Xinc); + disp('symmetrizing input'); +end + +% -- This doesn't work so well -- +% Xinc = tril(Xinc); +% [ii,jj,vv] = find(Xinc); +% ij = sub2ind( [n,n], ii, jj ); +% linop = @(varargin)samp_op( n, ii, jj, ij, varargin{:} ); +% +% Xinc = Xinc + tril(Xinc,-1)'; + +% This is better: +linop = linop_subsample(Xinc); +vv = full( Xinc( find(Xinc) ) ); + +% Extract the linear operators +[x,out,opts] = tfocs( smooth_quad, { linop, -vv }, proj_psdUTrace(tr), Xinc, opts ); + +% we use a special sampling operator to tell it to make it symmetric +function y = samp_op( n, ii, jj, ij, X, mode ) +switch mode + case 0, + y = { [n,n], [length(ii),1] }; + case 1, + X = (X+X')/2; + y = full(X(ij)); %otherwise it is a sparse vector + case 2, + y = sparse( ii, jj, X, n, n ); + y = y + tril(y,-1)'; +end + +function X = symmetrizeSparseMatrix( X ) + % Some entries were only specified once, others were + % double-specified, so we don't know if we should + % divide by 1 or 2 + ind1 = find( tril(X) ); + ind2 = find( tril(X',-1) ); + X = tril(X) + tril( X', -1); + ind = intersect(ind1,ind2); + X(ind) = X(ind)/2; + X = X + tril(X,-1)'; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/solver_sBP.m b/solver_sBP.m new file mode 100644 index 0000000..fe74849 --- /dev/null +++ b/solver_sBP.m @@ -0,0 +1,70 @@ +function varargout = solver_sBP( A, b, mu, x0, z0, opts, varargin ) +% SOLVER_SBP Basis pursuit (l1-norm with equality constraints). Uses smoothing. +% [ x, out, opts ] = solver_sBP( A, b, mu, x0, z0, opts ) +% Solves the smoothed basis pursuit problem +% minimize norm(x,1) + 0.5*mu*(x-x0).^2 +% s.t. A * x == b +% by constructing and solving the composite dual +% maximize - g_sm(z) +% where +% g_sm(z) = sup_x -norm(x,1)-(1/2)*mu*norm(x-x0) +% A must be a linear operator or matrix, and b must be a vector. The +% initial point x0 and the options structure opts are optional. +% +% If "nonneg" is a field in "opts" and opts.nonneg is true, +% then the constraints are A * x == b AND x >= 0 + +% Supply default values +error(nargchk(3,7,nargin)); +if nargin < 4, x0 = []; end +if nargin < 5, z0 = []; end +if nargin < 6, opts = []; end +if ~isfield( opts, 'restart' ), opts.restart = 400; end + + +% -- legacy options from original software -- +if isfield(opts,'lambda0') + opts = rmfield(opts,'lambda0'); +end +if isfield(opts,'xPlug') + opts = rmfield(opts,'xPlug'); +end +if isfield(opts,'solver') + svr = opts.solver; + opts = rmfield(opts,'solver'); + if isfield(opts,'alg') && ~isempty(opts.alg) + disp('Warning: conflictiong options for the algorithm'); + else + % if specified as "solver_AT", truncate: + s = strfind( svr, '_' ); + if ~isempty(s), svr = svr(s+1:end); end + opts.alg = svr; + end +end + + +nonneg = false; +if isfield(opts,'nonneg') + nonneg = opts.nonneg; + opts = rmfield(opts,'nonneg'); +end +if isfield(opts,'nonNeg') + nonneg = opts.nonNeg; + opts = rmfield(opts,'nonNeg'); +end + +if nonneg + % -- case: x >= 0 constraints + prox = prox_l1pos; +else + % -- case: no x >= 0 constraint + prox = prox_l1; +end +[varargout{1:max(nargout,1)}] = ... + tfocs_SCD( prox, { A, -b }, proj_Rn, mu, x0, z0, opts, varargin{:} ); + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_sBPDN.m b/solver_sBPDN.m new file mode 100644 index 0000000..11504da --- /dev/null +++ b/solver_sBPDN.m @@ -0,0 +1,83 @@ +function varargout = solver_sBPDN( A, b, epsilon, mu, x0, z0, opts, varargin ) +% SOLVER_SBPDN Basis pursuit de-noising. BP with relaxed constraints. Uses smoothing. +% [ x, out, opts ] = solver_sBPDN( A, b, epsilon, mu, x0, z0, opts ) +% Solves the smoothed basis pursuit denoising problem +% minimize norm(x,1) + 0.5*mu*(x-x0).^2 +% s.t. norm(A*x-b,2) <= epsilon +% by constructing and solving the composite dual +% maximize - g_sm(z) - epsilon*norm(z,2) +% where +% gsm(z) = sup_x -norm(x,1)-(1/2)*mu*norm(x-x0) +% A must be a linear operator or matrix, and b must be a vector. The +% initial point x0 and the options structure opts are optional. +% +% If "nonneg" is a field in "opts" and opts.nonneg is true, +% then the constraints are norm(A*x-b,2) <= epsilon AND x >= 0 + +% Supply default values +error(nargchk(4,8,nargin)); +if nargin < 5, x0 = []; end +if nargin < 6, z0 = []; end +if nargin < 7, opts = []; end +if ~isfield( opts, 'restart' ), opts.restart = 400; end + +if epsilon < 0 + error('TFOCS error: epsilon is negative'); +end +if ~epsilon + error('TFOCS error: cannot handle epsilon = 0. Please call solver_sBP instead'); +elseif epsilon < 100*builtin('eps') + warning('TFOCS:badConstraint',... + 'TFOCS warning: epsilon is near zero; consider calling solver_sBP instead'); +end + + +% -- legacy options from original software -- +if isfield(opts,'lambda0') + opts = rmfield(opts,'lambda0'); +end +if isfield(opts,'xPlug') + opts = rmfield(opts,'xPlug'); +end +if isfield(opts,'solver') + svr = opts.solver; + opts = rmfield(opts,'solver'); + if isfield(opts,'alg') && ~isempty(opts.alg) + disp('Warning: conflictiong options for the algorithm'); + else + % if specified as "solver_AT", truncate: + s = strfind( svr, '_' ); + if ~isempty(s), svr = svr(s+1:end); end + opts.alg = svr; + end +end + + + + +nonneg = false; +if isfield(opts,'nonneg') + nonneg = opts.nonneg; + opts = rmfield(opts,'nonneg'); +end +if isfield(opts,'nonNeg') + nonneg = opts.nonNeg; + opts = rmfield(opts,'nonNeg'); +end + +if nonneg + % -- case: x >= 0 constraints + prox = prox_l1pos; +else + % -- case: no x >= 0 constraint + prox = prox_l1; +end +[varargout{1:max(nargout,1)}] = ... + tfocs_SCD( prox, { A, -b }, prox_l2( epsilon ), mu, x0, z0, opts, varargin{:} ); + + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_sBPDN_W.m b/solver_sBPDN_W.m new file mode 100644 index 0000000..d799361 --- /dev/null +++ b/solver_sBPDN_W.m @@ -0,0 +1,68 @@ +function varargout = solver_sBPDN_W( A, W, b, epsilon, mu, x0, z0, opts, varargin ) +% SOLVER_SBPDN_W Weighted BPDN problem. Uses smoothing. +% [ x, out, opts ] = solver_sBPDN_W( A, W, b, epsilon, mu, x0, z0, opts ) +% Solves the smoothed basis pursuit denoising problem +% minimize norm(Wx,1) + 0.5*mu*(x-x0).^2 +% s.t. norm(A*x-b,2) <= epsilon +% by constructing and solving the composite dual. +% A and W must be a linear operator or matrix, and b must be a vector. The +% initial point x0 and the options structure opts are optional. +% See also solver_sBPDN + +% Supply default values +error(nargchk(5,9,nargin)); +if nargin < 6, x0 = []; end +if nargin < 7, z0 = []; end +if nargin < 8, opts = []; end +if ~isfield( opts, 'restart' ), opts.restart = 1000; end + +if epsilon < 0 + error('TFOCS error: epsilon is negative'); +end +% we don't have solver_sBP_W, so am building this into this function: +%if ~epsilon + %error('TFOCS error: cannot handle epsilon = 0. Please call solver_sBP_W instead'); +%elseif epsilon < 100*builtin('eps') + %warning('TFOCS:badConstraint',... + %'TFOCS warning: epsilon is near zero; consider calling solver_sBP instead'); +%end + +% Need to estimate the norms of A*A' and W*W' in order to be most efficient +if isfield( opts, 'noscale' ) && opts.noscale, + normA2 = 1; normW2 = 1; +else + normA2 = []; normW2 = []; + if isfield( opts, 'normA2' ), + normA2 = opts.normA2; + opts = rmfield( opts, 'normA2' ); + end + if isfield( opts, 'normW2' ), + normW2 = opts.normW2; + opts = rmfield( opts, 'normW2' ); + end +end +if isempty( normA2 ), + normA2 = linop_normest( A ).^2; +end +if isempty( normW2 ), + normW2 = linop_normest( W ).^2; +end + +% if ~isfield(opts,'L0') || isempty(opts.L0) +% opts.L0 = normA2/mu; % is this right? check +% end + +proxScale = sqrt( normW2 / normA2 ); +if epsilon > 0 + prox = { prox_l2( epsilon ), proj_linf(proxScale) }; +else + prox = { proj_Rn, proj_linf(proxScale) }; +end +W = linop_compose( W, 1 / proxScale ); +[varargout{1:max(nargout,1)}] = ... + tfocs_SCD( [], { A, -b; W, 0 }, prox, mu, x0, z0, opts, varargin{:} ); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_sBPDN_WW.m b/solver_sBPDN_WW.m new file mode 100644 index 0000000..7e92fa4 --- /dev/null +++ b/solver_sBPDN_WW.m @@ -0,0 +1,77 @@ +function varargout = solver_sBPDN_WW( A, alpha, W1, beta, W2, b, epsilon, mu, x0, z0, opts, varargin ) +% SOLVER_SBPDN_WW BPDN with two separate (weighted) l1-norm terms. Uses smoothing. +% [ x, out, opts ] = solver_sBPDN_WW( A, alpha, W_1, beta, W_2, b, epsilon, mu, x0, opts ) +% Solves the smoothed basis pursuit denoising problem +% minimize alpha*norm(W_1 x,1) + beta*norm(W_2 x, 1) 0.5*mu*(x-x0).^2 +% s.t. norm(A*x-b,2) <= epsilon +% by constructing and solving the composite dual. +% A, W_1 and W_2 must be a linear operator or matrix, and b must be a vector. The +% initial point x0 and the options structure opts are optional. +% See also solver_sBPDN and solver_sBPDN_W + +% Supply default values +error(nargchk(8,12,nargin)); +if nargin < 9, x0 = []; end +if nargin < 10, z0 = []; end +if nargin < 11, opts = []; end +if ~isfield( opts, 'restart' ), opts.restart = 5000; end + +if epsilon < 0 + error('TFOCS error: epsilon is negative'); +end +if ~epsilon + error('TFOCS error: cannot handle epsilon = 0. Please call solver_sBP instead'); +elseif epsilon < 100*builtin('eps') + warning('TFOCS:badConstraint',... + 'TFOCS warning: epsilon is near zero; consider calling solver_sBP instead'); +end + +% Need to estimate the norms of A*A' and W*W' in order to be most efficient +if isfield( opts, 'noscale' ) && opts.noscale, + normA2 = 1; normW12 = 1; normW22 = 1; +else + normA2 = []; normW12 = []; normW22 = []; + if isfield( opts, 'normA2' ) + normA2 = opts.normA2; + opts = rmfield( opts, 'normA2' ); + end + if isfield( opts, 'normW12' ) + normW12 = opts.normW12; + opts = rmfield( opts, 'normW12' ); + end + if isfield( opts, 'normW22' ) + normW22 = opts.normW22; + opts = rmfield( opts, 'normW22' ); + end +end +if isempty( normA2 ), + normA2 = linop_normest( A ).^2; +end +if isempty( normW12 ), + normW12 = linop_normest( W1 ).^2; +end +if isempty( normW22 ), + normW22 = linop_normest( W2 ).^2; +end +if isempty(alpha), + alpha = 1; +end +if isempty(beta), + beta = 1; +end + +proxScale1 = sqrt( normW12 / normA2 ); +proxScale2 = sqrt( normW22 / normA2 ); +prox = { prox_l2( epsilon ), ... + proj_linf( proxScale1 * alpha ),... + proj_linf( proxScale2 * beta ) }; +W1 = linop_compose( W1, 1 / proxScale1 ); +W2 = linop_compose( W2, 1 / proxScale2 ); +[varargout{1:max(nargout,1)}] = ... + tfocs_SCD( [], { A, -b; W1, 0; W2, 0 }, prox, mu, x0, z0, opts, varargin{:} ); + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_sDantzig.m b/solver_sDantzig.m new file mode 100644 index 0000000..1130d25 --- /dev/null +++ b/solver_sDantzig.m @@ -0,0 +1,84 @@ +function varargout = solver_sDantzig( A, b, delta, mu, x0, z0, opts, varargin ) +% SOLVER_SDANTZIG Dantzig selector problem. Uses smoothing. +%[ x, out, opts ] = solver_sDantzig( A, b, delta, mu, x0, z0, opts ) +% Solves the smoothed Dantzig +% minimize norm(x,1) + (1/2)*mu*norm(x-x0).^2 +% s.t. norm(D.*(A'*(A*x-b)),Inf) <= delta +% by constructing and solving the composite dual +% maximize - g_sm(z) - delta*norm(z,1) +% where +% gsm(z) = sup_x -norm(x,1)-(1/2)*mu*norm(x-x0) +% A must be a linear operator, b must be a vector, and delta and mu +% must be positive scalars. Initial points x0 and z0 are optional. +% The standard calling sequence assumes that D=I. To supply a scaling, +% pass the cell array { A, D } instead of A. D must either be a scalar, +% a vector of weights, or a linear operator. + +% Supply default values +error(nargchk(4,8,nargin)); +if nargin < 5, x0 = []; end +if nargin < 6, z0 = []; end +if nargin < 7, opts = []; end + +% -- legacy options from original software -- +if isfield(opts,'lambda0') + z0 = opts.lambda0; + opts = rmfield(opts,'lambda0'); +end +if isfield(opts,'xPlug') + x0 = opts.xPlug; + opts = rmfield(opts,'xPlug'); +end +if isfield(opts,'solver') + svr = opts.solver; + opts = rmfield(opts,'solver'); + if isfield(opts,'alg') && ~isempty(opts.alg) + disp('Warning: conflictiong options for the algorithm'); + else + % if specified as "solver_AT", truncate: + s = strfind( svr, '_' ); + if ~isempty(s), svr = svr(s+1:end); end + opts.alg = svr; + end +end + +% Extract the linear operators +D = []; +if isa( A, 'cell' ), + if length(A) > 1, D = A{2}; end + A = A{1}; +end +if isempty(D), + D = @(x)x; +elseif isa( D, 'double' ), + D = @(x)D.*x; +end +if isa( A, 'double' ), + A = linop_matrix(A); +end + +% Call TFOCS +objectiveF = prox_l1; +affineF = { @(y,mode)linear_DS( D, A, y, mode ), -D(A(b,2)) }; +dualproxF = prox_l1( delta ); +[varargout{1:max(nargout,1)}] = ... + tfocs_SCD( objectiveF, affineF, dualproxF, mu, x0, z0, opts, varargin{:} ); + +% Implements x -> D*A'*A*x and its adjoint if A is a linop +function y = linear_DS( D, A, y, mode ) +switch mode, +case 0, + y = A([],0); + if iscell( y ), + y = { y{1}, y{1} }; + else + y = { [y(2),1], [y(2),1] }; + end +case 1, y = D(A(A(y,1),2)); +case 2, y = A(A(D(y),1),2); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_sDantzig_W.m b/solver_sDantzig_W.m new file mode 100644 index 0000000..600067f --- /dev/null +++ b/solver_sDantzig_W.m @@ -0,0 +1,127 @@ +function varargout = solver_sDantzig_W( A,W, b, delta, mu, x0, z0, opts, varargin ) +% SOLVER_SDANTZIG_W Weighted Dantzig selector problem. Uses smoothing. +%[ x, out, opts ] = solver_sDantzig_W( A,W, b, delta, mu, x0, z0, opts ) +% Solves the smoothed Dantzig +% minimize norm(W*x,1) + (1/2)*mu*norm(x-x0).^2 +% s.t. norm(D.*(A'*(A*x-b)),Inf) <= delta +% by constructing and solving the composite dual +% +% A and W must be a linear operator, b must be a vector, and delta and mu +% must be positive scalars. Initial points x0 and z0 are optional. +% The standard calling sequence assumes that D=I. To supply a scaling, +% pass the cell array { A, D } instead of A. D must either be a scalar, +% a vector of weights, or a linear operator. +% +% Pass in the options "normA2" and "normW2" (which are ||A||^2 +% and ||W||^2 respectively) for best efficiency. +% +% See also solver_sDantzig + +% Supply default values +error(nargchk(5,9,nargin)); +if nargin < 6, x0 = []; end +if nargin < 7, z0 = []; end +if nargin < 8, opts = []; end + +if isfield(opts,'solver') + svr = opts.solver; + opts = rmfield(opts,'solver'); + if isfield(opts,'alg') && ~isempty(opts.alg) + disp('Warning: conflictiong options for the algorithm'); + else + % if specified as "solver_AT", truncate: + s = strfind( svr, '_' ); + if ~isempty(s), svr = svr(s+1:end); end + opts.alg = svr; + end +end + +% Extract the linear operators +D = []; +if isa( A, 'cell' ), + if length(A) > 1, D = A{2}; end + A = A{1}; +end +if isempty(D), + D = @(x)x; +elseif isa( D, 'double' ), + D = @(x)D.*x; +end +if isa( A, 'double' ), + % if "A" is not too rectangular, it is probably more efficient + % to compute A'*A once at the beginning and store it. + mn = min(size(A)); + mx = max(size(A)); + if mn >= .7*mx && mx < 1e5 + AA = @(y,mode)linear_DS_AA( D, A'*A, y, mode ); + A = linop_matrix(A); + else + A = linop_matrix(A); + AA = @(y,mode)linear_DS( D, A, y, mode ); + end +else + AA = @(y,mode)linear_DS( D, A, y, mode ); +end + +% Need to estimate the norms of A*A' and W*W' in order to be most efficient +if isfield( opts, 'noscale' ) && opts.noscale, + normA2 = 1; normW2 = 1; +else + normA2 = []; normW2 = []; + if isfield( opts, 'normA2' ), + normA2 = opts.normA2; + opts = rmfield( opts, 'normA2' ); + end + if isfield( opts, 'normW2' ), + normW2 = opts.normW2; + opts = rmfield( opts, 'normW2' ); + end +end +if isempty( normA2 ), + normA2 = linop_normest( A ).^2; +end +if isempty( normW2 ), + normW2 = linop_normest( W ).^2; +end + +% Call TFOCS +proxScale = sqrt( normW2 / normA2 ); +prox = { prox_l1( delta ); proj_linf(proxScale) }; +W = linop_compose( W, 1/proxScale); +affineF = {AA, -D(A(b,2)); W, 0 }; +[varargout{1:max(nargout,1)}] = ... + tfocs_SCD( [], affineF, prox, mu, x0, z0, opts, varargin{:} ); + + +% Implements x -> D*A'*A*x and its adjoint if A is a linop +function y = linear_DS( D, A, y, mode ) +switch mode, +case 0, + y = A([],0); + if iscell( y ), + y = { y{1}, y{1} }; + else + y = { [y(2),1], [y(2),1] }; + end +case 1, y = D(A(A(y,1),2)); +case 2, y = A(A(D(y),1),2); +end + +function y = linear_DS_AA( D, AA, y, mode ) +% similar to above, but expects AA to be an explicit Hermitian matrix +switch mode, +case 0, + y = size(AA); + if iscell( y ), + y = { y{1}, y{1} }; + else + y = { [y(2),1], [y(2),1] }; + end +case 1, y = D(AA*y); +case 2, y = AA*D(y); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_sLMI.m b/solver_sLMI.m new file mode 100644 index 0000000..5cf7c32 --- /dev/null +++ b/solver_sLMI.m @@ -0,0 +1,101 @@ +function varargout = solver_sLMI( A0, A, b, mu, x0, z0, opts, varargin ) +% SOLVER_SLMI Generic linear matrix inequality problems (LMI is the dual of a SDP). Uses smoothing. +% [ y, out, opts ] = solver_sLMI( A0, A, b, mu, y0, z0, opts ) +% Solves the smoothed Linear-Matrix Inequality (LMI) problem +% +% minimize_y b'*y +% s.t. A0 + sum_i A_i y(i) >= 0 +% " >= 0 " indicates that a matrix is positive semi-definite. +% +% "A0" and must be a symmetric/Hermitian matrix, "b" must be a vector, and +% "A" must be a matrix (dense or sparse) +% with the convention that each row of A stores the vectorized symmetric/Hermitian +% matrix A_i, so that sum_i A_i y(i) can be written as mat(A'*y) +% (where mat() reshapes a vector into a square matrix) +% if "A" is a function, then in forward mode it should compute A'*y +% and in transpose mode it should compute A*X +% +% Note: A0 and A_i must be symmetric/Hermitian, but this function +% does not check for it, so the user must check. If you get +% unexpected errors, this may be the culprit. +% +% For maximum efficiency, the user should specify the spectral norm of A, +% via opts.normA +% (e.g. opts.normA = norm(A) or opts.normA = normest(A)) +% +% By default, this assumes variables are real. +% To allow y to be complex, either pass in a complex value for y0 +% or make sure A or A0 is complex, +% or specify opts.cmode = 'R2C' +% (note: cmode = 'C2C' is not supported) +% +% See also solver_sSDP + +% Supply default values +error(nargchk(4,8,nargin)); +if nargin < 5, x0 = []; end +if nargin < 6, z0 = []; end +if nargin < 7, opts = []; end +if ~isfield( opts, 'restart' ), opts.restart = 1000; end + +% Do we automatically re-scale "A"? +% (there are two dual variables, one corresponding to Ax==b +% and one corresponding to I*x >= 0, and the dual problem +% is most efficient if norm(I) = norm(A), +% hence we rescale A <-- A/norm(A) ) +if isfield( opts, 'noscale' ) && opts.noscale + % The user has forced us not to automatically rescale + normA = 1; +else + normA = []; + if isfield( opts, 'normA' ), + normA = opts.normA; + opts = rmfield( opts, 'normA' ); + end +end +if isempty( normA ), + normA = linop_normest( A ); +end +if isfield( opts, 'noscale' ) + opts = rmfield(opts,'noscale'); +end + + +N = size(A0,1); if N ~= size(A0,2), error('"A0" must be square and symmetric'); end +if ~isa(A,'function_handle') + % We need to tell TFOCS that we'll be using matrix variables + M = size(A,1); if size(A,2) ~= N^2, error('"A" has wrong number of columns'); end + sz = { [M,1], [N,N] }; % specify dimensions of domain and range of A + if isfield( opts, 'cmode' ) + cmode = opts.cmode; + else + if ~isempty(x0) && ~isreal(x0) + cmode = 'R2C'; + elseif ~isreal(A) || ~isreal(A0) + cmode = 'R2C'; % if A is complex, then X must be, in order to get real output + else + cmode = 'R2R'; % default assumption + end + end + vec = @(x) x(:); + mat = @(y) reshape(y,N,N); + A = linop_handles( sz, @(y) mat(A'*y), @(X)real(A*vec(X)), cmode); +end + + + +% Perform the re-scaling: +A = linop_compose( A, 1 / normA ); +A0 = A0/normA; + + +obj = smooth_linear(b); + +[varargout{1:max(nargout,1)}] = ... + tfocs_SCD( obj, {A,A0}, {proj_psd}, mu, x0, z0, opts, varargin{:} ); + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_sLP.m b/solver_sLP.m new file mode 100644 index 0000000..bec94ec --- /dev/null +++ b/solver_sLP.m @@ -0,0 +1,82 @@ +function varargout = solver_sLP( c, A, b, mu, x0, z0, opts, varargin ) +% SOLVER_SLP Generic linear programming in standard form. Uses smoothing. +% [ x, out, opts ] = solver_sLP( c, A, b, mu, x0, z0, opts ) +% Solves the smoothed standard form Linear Program (LP) +% minimize c'*x + 0.5*mu*||x-x0||_2^2 +% s.t. A * x == b and x >= 0 +% +% "c" and "b" must be vectors, and "A" must be a matrix (dense or sparse) +% or a function that computes A*x and A'*y (see help documentation). +% +% If the constraint "x >= 0" is not needed, then specify this by +% setting: +% opts.nonnegativity = false +% +% For maximum efficiency, the user should specify the spectral norm of A, +% via opts.normA +% (e.g. opts.normA = norm(A) or opts.normA = normest(A)) + +% Supply default values +error(nargchk(4,8,nargin)); +if nargin < 5, x0 = []; end +if nargin < 6, z0 = []; end +if nargin < 7, opts = []; end +if ~isfield( opts, 'restart' ), opts.restart = 1000; end + + +% Do we include the x >= 0 constraint? +NONNEG = true; +if isfield(opts,'nonnegativity') + if ~opts.nonnegativity + NONNEG = false; + end + opts = rmfield(opts,'nonnegativity'); +end + + +% Do we automatically re-scale "A"? +% (there are two dual variables, one corresponding to Ax==b +% and one corresponding to I*x >= 0, and the dual problem +% is most efficient if norm(I) = norm(A), +% hence we rescale A <-- A/norm(A) ) +if isfield( opts, 'noscale' ) && opts.noscale + % The user has forced us not to automatically rescale + normA = 1; +else + normA = []; + if isfield( opts, 'normA' ), + normA = opts.normA; + opts = rmfield( opts, 'normA' ); + end +end +if isempty( normA ), + normA = linop_normest( A ); +end +if isfield( opts, 'noscale' ) + opts = rmfield(opts,'noscale'); +end + +% Perform the re-scaling: +if isnumeric(A), A = A/normA; % do it once +else +A = linop_compose( A, 1 / normA ); +end +b = b/normA; + + +obj = smooth_linear(c); + +if ~NONNEG + % There is no x >= 0 constraint: + [varargout{1:max(nargout,1)}] = ... + tfocs_SCD( obj, { A, -b }, proj_Rn, mu, x0, z0, opts, varargin{:} ); +else + % There is a x >= 0 constraint: + [varargout{1:max(nargout,1)}] = ... + tfocs_SCD( obj, { 1,0;A,-b}, {proj_Rplus,proj_Rn}, mu, x0, z0, opts, varargin{:} ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_sLP_box.m b/solver_sLP_box.m new file mode 100644 index 0000000..d3ca903 --- /dev/null +++ b/solver_sLP_box.m @@ -0,0 +1,83 @@ +function varargout = solver_sLP_box( c, A, b, l, u, mu, x0, z0, opts, varargin ) +% SOLVER_SLP_BOX Generic linear programming with box constraints. Uses smoothing. +% [ x, out, opts ] = solver_sLP_box( c, A, b, l, u, mu, x0, z0, opts ) +% Solves the smoothed standard form Linear Program (LP) with box-constraints +% minimize c'*x + 0.5*mu*||x-x0||_2^2 +% s.t. A * x == b and l <= x <= b +% +% "c" and "b" must be vectors, and "A" must be a matrix (dense or sparse) +% or a function that computes A*x and A'*y (see help documentation). +% "l" and "u" are vectors, or scalars (in which case they are a scalar +% times the vector of all ones ) +% +% If only "l" (or only "u") is needed, set "u" (or "l") to the empty matrix []. +% +% For maximum efficiency, the user should specify the spectral norm of A, +% via opts.normA +% (e.g. opts.normA = norm(A) or opts.normA = normest(A)) +% +% See also solver_sLP + +% Supply default values +error(nargchk(6,10,nargin)); +if nargin < 7, x0 = []; end +if nargin < 8, z0 = []; end +if nargin < 9, opts = []; end +if ~isfield( opts, 'restart' ), opts.restart = 1000; end + + + +% Do we automatically re-scale "A"? +% (there are two dual variables, one corresponding to Ax==b +% and one corresponding to I*x >= 0, and the dual problem +% is most efficient if norm(I) = norm(A), +% hence we rescale A <-- A/norm(A) ) +if isfield( opts, 'noscale' ) && opts.noscale + % The user has forced us not to automatically rescale + normA = 1; +else + normA = []; + if isfield( opts, 'normA' ), + normA = opts.normA; + opts = rmfield( opts, 'normA' ); + end +end +if isempty( normA ), + normA = linop_normest( A ); +end +if isfield( opts, 'noscale' ) + opts = rmfield(opts,'noscale'); +end + +% Perform the re-scaling: +if isnumeric(A), A = A/normA; % do it once +else +A = linop_compose( A, 1 / normA ); +end +b = b/normA; + + +obj = smooth_linear(c); + +if isempty(l) && isempty(u) + % There is no x >= 0 constraint: + [varargout{1:max(nargout,1)}] = ... + tfocs_SCD( obj, { A, -b }, proj_Rn, mu, x0, z0, opts, varargin{:} ); +elseif isempty(u) + % There is a x >= l constraint, i.e. x - l >= 0 + [varargout{1:max(nargout,1)}] = ... + tfocs_SCD( obj, { 1,-l;A,-b}, {proj_Rplus,proj_Rn}, mu, x0, z0, opts, varargin{:} ); +elseif isempty(l) + % There is a x <= u constraint, i.e. -x + u >= 0 + [varargout{1:max(nargout,1)}] = ... + tfocs_SCD( obj, {-1, u;A,-b}, {proj_Rplus,proj_Rn}, mu, x0, z0, opts, varargin{:} ); +else + % have both upper and lower box constraints + [varargout{1:max(nargout,1)}] = ... + tfocs_SCD( obj, {1,-l;-1, u;A,-b}, {proj_Rplus,proj_Rplus,proj_Rn}, mu, x0, z0, opts, varargin{:} ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_sNuclearBP.m b/solver_sNuclearBP.m new file mode 100644 index 0000000..5ab28bd --- /dev/null +++ b/solver_sNuclearBP.m @@ -0,0 +1,107 @@ +function varargout = solver_sNuclearBP( omega, b, mu, x0, z0, opts, varargin ) +% SOLVER_SNUCLEARBP Nuclear norm basis pursuit problem (i.e. matrix completion). Uses smoothing. +% [ x, out, opts ] = solver_sNuclearBP( omega, b, mu, X0, Z0, opts ) +% Solves the smoothed nuclear norm basis pursuit problem +% minimize norm_nuc(X) + 0.5*mu*norm(X-X0,'fro').^2 +% s.t. A_omega * x == b +% by constructing and solving the composite dual +% maximize - g_sm(z) +% where +% g_sm(z) = sup_x -norm(x,1)-(1/2)*mu*norm(x-x0) +% A_omega is the restriction to the set omega, and b must be a vector. The +% initial point x0 and the options structure opts are optional. +% +% The "omega" term may be in one of three forms: +% (1) OMEGA, a sparse matrix. Only the nonzero pattern is important. +% (2) {n1,n2,omega}, a cell, where [n1,n2] = size(X), and omega +% is the vector of linear indices of the observed set +% (3) {n1,n2,omegaI,omegaJ}, a cell. Similar to (2), except the set +% omega is now specified by subscripts. Specifically, +% omega = sub2ind( [n1,n2], omegaI, omegaJ) and +% [omegaI,omegaJ] = ind2sub( [n1,n2], omega ) +% +% If the options field "largeScale" is provided and set to true, then uses +% a Lanczos-based SVD + +% Supply default values +error(nargchk(3,7,nargin)); +if nargin < 4, x0 = []; end +if nargin < 5, z0 = []; end +if nargin < 6, opts = []; end +if ~isfield( opts, 'restart' ), + opts.restart = 50; +end + +if isempty(omega) + error( 'Sampling operator cannot be empty.' ); +elseif issparse(omega) + [omegaI,omegaJ] = find(omega); + [n1,n2] = size(omega); + omega_lin = sub2ind( [n1,n2], omegaI, omegaJ ); +elseif iscell(omega) + switch length(omega) + case 3, + [ n1, n2, omega_lin ] = deal( omega{:} ); + [omegaI,omegaJ] = ind2sub( [n1,n2], omega_lin ); + case 4 + [ n1, n2, omegaI, omegaJ ] = deal( omega{:} ); + omega_lin = sub2ind( [n1,n2], omegaI, omegaJ ); + otherwise + error( 'Incorrect format for the sampling operator.' ); + end +else + error( 'Incorrect format for the sampling operator.' ); +end +nnz = numel(omega_lin); +omega_lin = omega_lin(:); % make it a column vector +b = b(:); % make it a column vector +if ~isequal( size(b), [ nnz, 1 ] ), + error( 'Incorrect size for the sampled data.' ); +end + +% automatically do kicking on the dual variable: +if isempty(z0) + z0 = sparse(omegaI,omegaJ,b,n1,n2); + nY = linop_normest(z0,'R2R',1e-4,300); + z0 = b/nY; +end + +packSVDflag = isa(x0,'packSVD'); +% Note: packSVD is not supported, so has been removed +A = @(varargin)linop_nuclear( n1, n2, nnz, omega_lin, omegaI, omegaJ,packSVDflag, varargin{:} ); + +q=1; +if isfield(opts,'largeScale') && opts.largeScale + prox = prox_nuclear(q,true); +else + prox = prox_nuclear(q); +end +if isfield(opts,'largeScale'), opts = rmfield(opts,'largeScale'); end + +[varargout{1:max(nargout,1)}] = ... + tfocs_SCD( prox, { A, -b }, proj_Rn, mu, x0, z0, opts, varargin{:} ); + +% +% Implements the matrix sampling operator: X -> [X_ij]_{i,j\in\omega} +% +function y = linop_nuclear( n1, n2, nnzs, omega, omegaI, omegaJ, packSVDflag, x, mode ) +switch mode, + case 0, + y = { [n1,n2], [nnzs,1] }; + case 1, +% y = x(omega); + S = []; + S.type = '()'; + S.subs = {omega}; + y = subsref(x,S); + case 2, + y = sparse( omegaI, omegaJ, x, n1, n2 ); + if packSVDflag + y = packSVD(y); + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_sNuclearBPDN.m b/solver_sNuclearBPDN.m new file mode 100644 index 0000000..3310c6b --- /dev/null +++ b/solver_sNuclearBPDN.m @@ -0,0 +1,78 @@ +function varargout = solver_sNuclearBPDN( omega, b, epsilon, mu, x0, z0, opts, varargin ) +% SOLVER_SNUCLEARBPDN Nuclear norm basis pursuit problem with relaxed constraints. Uses smoothing. +% [ x, out, opts ] = solver_sNuclearBPDN( omega, b, epsilon,mu, X0, Z0, opts ) +% Solves the smoothed nuclear norm basis pursuit problem +% minimize norm_nuc(X) + 0.5*mu*norm(X-X0,'fro').^2 +% s.t. ||A_omega * x - b || <= epsilon +% by constructing and solving the composite dual +% maximize - g_sm(z) +% where +% g_sm(z) = sup_x -norm(x,1)-(1/2)*mu*norm(x-x0) +% A_omega is the restriction to the set omega, and b must be a vector. The +% initial point x0 and the options structure opts are optional. +% +% The "omega" term may be in one of three forms: +% (1) OMEGA, a sparse matrix. Only the nonzero pattern is important. +% (2) {n1,n2,omega}, a cell, where [n1,n2] = size(X), and omega +% is the vector of linear indices of the observed set +% (3) {n1,n2,omegaI,omegaJ}, a cell. Similar to (2), except the set +% omega is now specified by subscripts. Specifically, +% omega = sub2ind( [n1,n2], omegaI, omegaJ) and +% [omegaI,omegaJ] = ind2sub( [n1,n2], omega ) + +% Supply default values +error(nargchk(4,8,nargin)); +if nargin < 5, x0 = []; end +if nargin < 6, z0 = []; end +if nargin < 7, opts = []; end +if ~isfield( opts, 'restart' ), + opts.restart = 50; +end + +if isempty(omega) + error( 'Sampling operator cannot be empty.' ); +elseif issparse(omega) + [omegaI,omegaJ] = find(omega); + [n1,n2] = size(omega); + omega_lin = sub2ind( [n1,n2], omegaI, omegaJ ); +elseif iscell(omega) + switch length(omega) + case 3, + [ n1, n2, omega_lin ] = deal( omega{:} ); + [omegaI,omegaJ] = ind2sub( [n1,n2], omega_lin ); + case 4 + [ n1, n2, omegaI, omegaJ ] = deal( omega{:} ); + omega_lin = sub2ind( [n1,n2], omegaI, omegaJ ); + otherwise + error( 'Incorrect format for the sampling operator.' ); + end +else + error( 'Incorrect format for the sampling operator.' ); +end +nnz = numel(omega_lin); +if ~isequal( size(b), [ nnz, 1 ] ), + error( 'Incorrect size for the sampled data.' ); +end + +% TODO: see the new linop_subsample.m file +A = @(varargin)linop_nuclear( n1, n2, nnz, omega_lin, omegaI, omegaJ, varargin{:} ); +[varargout{1:max(nargout,1)}] = ... + tfocs_SCD( prox_nuclear, { A, -b }, prox_l2(epsilon), mu, x0, z0, opts, varargin{:} ); + +% +% Implements the matrix sampling operator: X -> [X_ij]_{i,j\in\omega} +% +function y = linop_nuclear( n1, n2, nnz, omega, omegaI, omegaJ, x, mode ) +switch mode, + case 0, + y = { [n1,n2], [nnz,1] }; + case 1, + y = x(omega); + case 2, + y = sparse( omegaI, omegaJ, x, n1, n2 ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/solver_sSDP.m b/solver_sSDP.m new file mode 100644 index 0000000..9af5bd0 --- /dev/null +++ b/solver_sSDP.m @@ -0,0 +1,104 @@ +function varargout = solver_sSDP( c, A, b, mu, x0, z0, opts, varargin ) +% SOLVER_SSDP Generic semi-definite programs (SDP). Uses smoothing. +% [ x, out, opts ] = solver_sSDP( C, A, b, mu, X0, z0, opts ) +% Solves the smoothed standard form Semi-Definite Program (SDP) +% minimize trace(C'*X) + 0.5*mu*||X-X0||_F^2 +% s.t. A * vec(X) == b and X >= 0 +% where X >= 0 indicates that X is positive semi-definite. +% +% "C" and must be a matrix, "b" must be a vector, and +% "A" must be a matrix (dense or sparse) +% or a function that computes A*x and A'*y (see help documentation). +% The operation b = A*vec(X) is equivalent to +% b_i = trace( A_i'*X ) for i = 1:size(A,1), where A_i is the symmetric +% matrix formed by reshaping the ith row of A. +% +% +% For maximum efficiency, the user should specify the spectral norm of A, +% via opts.normA +% (e.g. opts.normA = norm(A) or opts.normA = normest(A)) +% +% By default, this assumes variables are real. +% To allow X to be complex, either pass in a complex value for X0 +% or make sure A is complex, +% or specify opts.cmode = 'C2R' +% (note: cmode = 'C2C' is not supported) +% +% See also solver_sLMI + +% Supply default values +error(nargchk(4,8,nargin)); +if nargin < 5, x0 = []; end +if nargin < 6, z0 = []; end +if nargin < 7, opts = []; end +if ~isfield( opts, 'restart' ), opts.restart = 1000; end + +% Do we automatically re-scale "A"? +% (there are two dual variables, one corresponding to Ax==b +% and one corresponding to I*x >= 0, and the dual problem +% is most efficient if norm(I) = norm(A), +% hence we rescale A <-- A/norm(A) ) +if isfield( opts, 'noscale' ) && opts.noscale + % The user has forced us not to automatically rescale + normA = 1; +else + normA = []; + if isfield( opts, 'normA' ), + normA = opts.normA; + opts = rmfield( opts, 'normA' ); + end +end +if isempty( normA ), + normA = linop_normest( A ); +end +if isfield( opts, 'noscale' ) + opts = rmfield(opts,'noscale'); +end + + +N = size(c,1); if N ~= size(c,2), error('"C" must be square and symmetric'); end +if ~isa(A,'function_handle') + % We need to tell TFOCS that we'll be using matrix variables + M = size(A,1); if size(A,2) ~= N^2, error('"A" has wrong number of columns'); end + sz = { [N,N], [M,1] }; % specify dimensions of domain and range of A + if isfield( opts, 'cmode' ) + cmode = opts.cmode; + else + if ~isempty(x0) && ~isreal(x0) + cmode = 'C2R'; + elseif ~isreal(A) || ~isreal(c) % bug fix, Tue Apr 26, 2011 + cmode = 'C2R'; % if A is complex, then X must be, in order to get real output + else + cmode = 'R2R'; % default assumption + end + end + vec = @(x) x(:); + mat = @(y) reshape(y,N,N); + A = linop_handles( sz, @(X)real(A*vec(X)), @(y) mat(A'*y),cmode); +end + + + +% Perform the re-scaling: +A = linop_compose( A, 1 / normA ); +b = b/normA; + + +obj = smooth_linear(c); + +[varargout{1:max(nargout,1)}] = ... + tfocs_SCD( obj, { 1,0;A,-b}, {proj_psd,proj_Rn}, mu, x0, z0, opts, varargin{:} ); + +% and undo the scaling by normA: +if nargout >= 2 && isfield( varargout{2},'dual' ) && normA ~= 1 + if isa( varargout{2}.dual,'tfocs_tuple') + varargout{2}.dual = tfocs_tuple( {varargout{2}.dual{1}, varargout{2}.dual{2}/normA }); + else + varargout{2}.dual{2} = varargout{2}.dual{2}/normA; + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/test_nonsmooth.m b/test_nonsmooth.m new file mode 100644 index 0000000..4eacf1b --- /dev/null +++ b/test_nonsmooth.m @@ -0,0 +1,219 @@ +function varargout = test_nonsmooth( f, N, DOM ) +% TEST_NONSMOOTH Runs diagnostic tests to ensure a non-smooth function conforms to TFOCS conventions +% TEST_NONSMOOTH( F ) +% tests whether the function handle F works as a tfocs non-nonsmooth +% function object. +% +% Requirements: +% f = F(X) must return the value of the function at X. +% p = F(X,t) must return the proximity operator of F at X, +% i.e. p = argmin_y F(y) + 1/(2*t)||y-x||^2 +% +% For an example, see PROX_L1.M +% +% TEST_NONSMOOTH( F, N ) +% specifies the size of the domain space. If N is a scalar, +% then the domain space is the set of N x 1 vectors. +% If N is a matrix of the form [n1, n2], the the domain +% space is the set of n1 x n2 matrices. +% +% TEST_NONSMOOTH( ..., DOM ) +% specifies the domain of F. DOM can be of the form [a,b] +% which signifies the 1D set (a,b). +% +% OK = TEST_NONSMOTH(...) +% returns "true" if all tests are passed, and "false" otherwise. +% +% Example: +% test_nonsmooth( prox_l1 ) +% +% See also private/tfocs_prox, prox_l1 + +OK = false; +PLOT = true; + +error(nargchk(1,3,nargin)); + +if ~isa(f,'function_handle') + fail('TFOCS nonsmooth function must be a FUNCTION HANDLE'); + return; +end +fprintf('== Testing the nonnonsmooth function %s ==\n', func2str(f) ); + +if nargin < 2 || isempty(N), N = 10; end +if nargin < 3 || isempty(DOM), DOM = [-1,1]; end + +a = DOM(1); b = DOM(2); + +x = (a+b)/2; +fprintf('Testing scalar inputs... \n'); +try + vi = f(x); +catch + fail('TFOCS nonsmooth function failed to return a function value'); + return; +end +if isinf(vi) + fail('TFOCS nonsmooth function tester: default domain is invalid. Please specify valid domain'); + return; +end +fprintf('\t\t\t\t...passed. \n'); + +% Now, also ask for the prox +fprintf('Testing proximity operator output... \n'); +t = 1; +try + [vi,gi] = f(x,t); +catch + fail('TFOCS nonsmooth function failed to compute a valid proximity operator'); + return; +end +fprintf('\t\t\t\t...passed. \n'); + + +% find a reasonable value of t: +x = a + .9*(b-a); +[vi,gi] = f(x,t); +cntr = 0; +% This is only good for proximity functions: for projections, +% the value of "t" has no effect: +if strfind( func2str(f), 'proj' ) + disp('Looks like this is a projection: this test may not work correctly.'); +else + disp('Looks like this is a proximity operator'); + while norm(gi - x )/norm(x) > .8 && cntr < 20 + t = t/2; + cntr = cntr + 1; + [vi,gi] = f(x,t); + end +end +% for t = 0, we have gi = x +% for t = Inf, gi is independent of x (i.e. gi = argmin_x f(x), +% which, for f(x) = ||x||, is gi = 0 ). + + + +% Now, try a vector +if isscalar(N) + x = repmat(x,N,1); +else + x = repmat(x,N(1),N(2)); + % if a > 0, assume we want a PSD matrix: + if a >= 0 + x = ones(N(1),N(2)) + eye(N(1),N(2) ); + end +end +fprintf('Testing vector inputs... \n'); +try + vi = f(x); +catch + fail('TFOCS nonsmooth function failed to compute f when supplied a vector input'); + return; +end +try + [vi,gi] = f(x,t); +catch + fail('TFOCS nonsmooth function failed to compute prox_f when supplied a vector input'); + return; +end +if ~isscalar(vi) + fail('TFOCS nonsmooth function value must be a scalar'); + return; +end +if ~all( size(gi) == size(x) ) + fail('TFOCS nonsmooth proximity function value must be same size as input'); + return; +end +fprintf('\t\t\t\t...passed. \n'); + +% gi = prox_{f,t}(x) +% let's verify that gi is actually a minimizer, by +% plotting the values for a few nearby points +s = min(t,.9*x(1,1) ); +x = x + .1/norm(x,Inf)*randn(size(x)); +[vi,gi] = f(x,s); +objective = @(y) f(y) + 1/(2*s)*norm(y-x,'fro')^2; +fprintf('Verifying the validity of the proximity calculation...\n'); +objGi = objective(gi); +fprintf('\tprox(x) has objective value: %.3e\n', objGi ); +sc = 1/norm(gi,'inf'); +OK_objective = true; +for t = 1:20 + randPt = gi + .01*sc*randn(size(gi)); + obj = objective(randPt); + fprintf('\tRandom pt #%2d has larger objective value by %7.2e', t,obj-objGi); + if obj < objGi + fprintf(' -- SMALLER!! This shouldn''t happen'); + OK_objective = false; + end + fprintf('\n'); +end +if ~OK_objective + fail('TFOCS nonsmooth: incorrect calculation of proximity function'); + return; +end +fprintf('\t\t\t\t...passed. \n'); + + +% 1D example. Does not require function to be vectorized +n = 200; % number of grid points +h = (b-a)/n; +grid = (a+h/2):h:(b-h/2); +% include "0" so things look nice and pointy: +if a < 0 && b > 0 + grid = unique( [grid,0] ); +end +% if isscalar(N) +% grid = repmat( grid, N, 1 ); +% % grid( 2:end, : ) = 0.5*grid( 2:end, : ); +% grid( 2:end, : ) = 0; +% end + +n = size(grid,2); +v = zeros(1,n); +g = zeros(1,n); +for i = 1:length(grid) + v(i) = f(grid(1,i) ); + [vi,gi] = f(grid(:,i), t ); + g(i) = gi(1); +end +grid = grid(1,:); +if PLOT + figure; + clf; + plot(grid,v,'-','linewidth',2); + hold all + plot(grid,g,'-','linewidth',2); + Y = get(gca,'ylim'); + Y(2) = 1.1*Y(2); + set(gca,'ylim',Y); + + legend('function','proximity operator'); +% title(func2str(f),'interpreter','none'); + title(sprintf('function, and proximity operator, with parameter t = %.3f',t)) + line([a,b], 0*[1,1],'color','k' ) + if a < 0 && b > 0 + line( 0*[1,1], get(gca,'ylim'),'color','k' ); + end + % show the y = x line + line( [a,b], [a,b], 'linestyle','--','color','k'); + +end + +OK = true; + + +disp('Test passed succesfully.'); +if nargout > 0 + varargout{1} = OK; +end + + + +function fail(str) + disp('Test failed. Reason:'); + disp(str); + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/test_proxPair.m b/test_proxPair.m new file mode 100644 index 0000000..461095c --- /dev/null +++ b/test_proxPair.m @@ -0,0 +1,213 @@ +function varargout = test_proxPair( f, g, N, nTrials, force_gama,tol,break_on_bad ) +% TEST_PROXPAIR Runs diagnostics on a pair of functions to check if they are Legendre conjugates. +% maxEr = test_proxPair( f, g ) +% will run tests to determine if f and g are really +% Legendre-Fenchel conjugates. +% f and g should be in TFOCS form (see examples below) +% which means they not only compute a function value f(x) +% but that they can also return the proximity operator. +% The test points are assumed to be scalars by default (N=1) +% +% ... = test_proxPair( f, g, N ) +% Same test, but now the test points will be vectors of length N +% ... = test_proxPair( f, g, x0 ) +% Test points will have the same size as x0. If x0 is a square matrix +% other than the zero or identity matrix, the domain will be the set of symmetric +% matrices if x0 is symmetric. If x0 is positive semidefinite, domain is taken +% to be the set of positive semidefinite matrices. +% ... = test_proxPair( f, g, [N or x0], nTrials ) +% will run "nTrials" (default: 10) +% ... = test_proxPair( f, g, [N or x0], nTrials, t ) +% will use the proximity operator: +% prox_f(v) = argmin f(x) + 1/(2t) *||x-v||^2 +% +% If the function works for t = 1 but not for other t, +% this helps you track down the bug (perhaps there is a t vs 1/t confusion +% somewhere...) +% +% (default: t>0 is chosen randomly on every trial ) +% +% ... = test_proxPair( f, g, [N or x0], nTrials, t, tol ) +% sets the tolerance level for considering an inequality to be violated +% (default is 1e-8) +% [x_bad, gamma_bad] = test_proxPair( ..., 'break' ) +% will return the offending point in the case that one of the the inequalities +% has been violated. If no inequalities are violated, then the empty +% matrices are returned. +% +% Note: TFOCS uses the dual function composed with the negative identity operator, +% so you may wish to use prox_scale( g, -1) instead of just g. +% +% Examples of valid pairs (f,g) (where q is any positive scalar, r is any scalar): +% +% f = prox_l1(q); and g = proj_linf(q); +% f = proj_l1(q); and g = prox_linf(q); +% f = proj_l2(q); and g = prox_l2(q); +% f = prox_hinge(q,r); and g = prox_hingeDual(q,r); +% f = proj_Rn; and g = proj_0; +% f = proj_Rplus; and g = proj_Rplus(-1); +% f = proj_nuclear; and g = prox_spectral; +% f = prox_nuclear; and g = proj_spectral; +% f = prox_trace; and g = proj_spectral(1,'symm') for X >= 0 +% f = proj_psdUTrace;and g = prox_spectral(1,'symm') fr X >= 0 +% +% Recall the definition of the conjugate function: +% g(y) = f^*(y) = sup_x - f(x) +% f(x) = sup_y - f^*(y) +% We use Moreau's Decomposition to generate identities. See Combettes and Wajs '05, +% lemma 2.10, http://www.ann.jussieu.fr/~plc/mms1.pdf +% Moreau's decomposition is: +% for all x, x = prox_( gamma*f )( x ) + gamma*prox_( g/gamma )( x/gamma ) +% +% and TFOCS uses (since t > 0) +% prox(f)(t)(x) := argmin f(v) + 1/(2t)||v-x||^2 +% = argmin t*f(v) + 1/2 *||v-x||^2 [ N.B. This is NOT 1/t*argmin... ] +% = prox_( t*f )( x ) +% +% so the identity in terms of TFOCS prox is: +% x = prox(f)(gamma)(x) + gamma*prox(g)(1/gamma)(x/gamma) ( ID 1 ) +% = xf + xg +% +% We also have the identity: +% f(xf) + g(xg/gamma) = / gamma ( ID 2 ) +% where xf and xg defined above. +% [ f(xf) + g(xg/gamma) >= /gamma ] is the Fenchel-Young inequality. +% +% The final identity is: +% ||x||^2/2 = gamma[ f^(gamma)(x) + g^(1/gamma)(x/gamma) ] ( ID 3 ) +% where +% f^t(x) = min f(v) + 1/(2t)||v-x||^2 +% +% These three identities are the three columns of errors output by this test function. +% See also prox_dualize.m + + +error(nargchk(2,7,nargin)); +if nargin < 3 || isempty(N), N = 1; end +if ~isscalar(N) + x0 = N; + % Check: if x0 is a symmetric matrix, then we assume domain + % is set of symmetric matrices. If it is also positive semidefinite, + % we assume domain is set of symmetric positive semidefinite matrices + % (but if x0 is all zeros or the identity, then do not make any assumptions) + [m,n] = size(x0); + if m == n && norm( x0-x0', 'fro' ) < 1e-10 && ... + norm( x0-eye(n),'fro') > 1e-10 && norm(x0,'fro') > 1e-10 + % it is symmetric + if min(eig(x0)) >= -1e-14 + fprintf('Assuming domain is set of positive semidefinite matrcies\n'); + symm = @(X) X*X'; + else + fprintf('Assuming domain is set of Hermitian matrices\n'); + symm = @(X) (X+X')/2; + end + else + symm = @(x) x; % do not symmetrize + end + if issparse(x0) + fprintf('Assuming domain is set of sparse matrices\n'); + newX = @() 100*symm(sprandn( m,n,.01 )); + else + newX = @() 100*symm(randn( size(N) )); + end + fprintf('Using a domain of size %d x %d\n', m, n ); +else + fprintf('Using a domain of size %d x %d\n', N, 1 ); + newX = @() 100*randn( N, 1 ); +end + +if nargin < 4 || isempty(nTrials), nTrials = 10; end +if nargin < 5 || isempty(force_gama), force_gama = false; end +% Note: "gamma" mispelled on purpose, since Matlab's "gamma" function +% creates problems when variables are named "gamma" +if nargin < 6 || isempty(tol), tol = 1e-8; end +if nargin < 7, break_on_bad=[];end +if strfind( lower(break_on_bad), 'break') + break_on_bad = true; +else + break_on_bad = false; +end + +fprintf('\n'); +maxEr = 0; +FenchelYoungViolation = false; +vec = @(x) x(:); +myDot = @(x,y) x(:)'*y(:); +for k = 1:nTrials + if force_gama + gama = double(force_gama); + else + gama = rand(1); + end + x = newX(); + [vf,xf] = f(x,gama); + [vg,xg] = g(x/gama,1/gama); + if numel(vf) > 1, error('The "f" objective function must be scalar valued'); end + if numel(vg) > 1, error('The "g" objective function must be scalar valued'); end + if isinf(vf) || isinf(vg) || isnan(vf) || isnan(vg) + error('Found either Inf or Nan values'); + % it is OK for f(x) to be Inf (i.e. for an indicator function) + % but not for f(x,t) to be Inf, since this is a projection + % or proximity operator + end + + xg = xg*gama; + + er = (xf+xg) - x; + + % Test the scalar identity: + % er1 = f( xf ) + g( xg/gama ) - xf'*xg/gama; % this can be inaccurate due to finite precision + er2 = vf + vg - myDot(xf,xg)/gama; % should be same as er2 + % Sept 3 2012, when vf is very large, we lose precision. Need to make + % this have *relative* precision + er2 = abs(er2/max( [abs(vf),abs(vg),1e-3] )); + + % And the other scalar identity: + % ||x||^2/2 = gama( f^(gama)(x) + g^(1/gama)(x/gama) ) + rhs = ( vf + 1/(2*gama)*norm(vec(xf-x))^2 ) + (vg + gama/2*norm(vec(xg/gama-x/gama))^2 ); + lhs = norm(vec(x) )^2/2/gama; + er3 = abs(rhs-lhs)/abs(lhs); + + + % Another test: does the Fenchel-Young inequality hold? (this test is unlikely to find violations) + % for all x, y, f(x) + g(y) >= x'*y + y = newX(); + vf = f(x); + vg = g(y); + if vf + vg < x'*y + FenchelYoungViolation = true; + end + + % when both f and g are projections, xf and xg should be orthogonal + fprintf('Random trial #%2d, errors are:\t%.2e,\t%.2e,\t%.2e\n', k, norm(er), er2, er3 ); + maxEr = max([norm(er),er2,er3,maxEr] ); + + if break_on_bad && maxEr > tol + varargout{1} = x; + varargout{2} = gama; + disp('Found a bad pair: terminating early'); + return; + end +end + + +fprintf('Worst error was %.2e\n', maxEr ); +if maxEr > tol + disp('This is a BAD sign -- the functions are either not correctly implemented or there is a lot of roundoff error'); + disp(' Try running this again forcing t=1" to help find the source of error (see help file)'); +else + disp('This is a GOOD sign -- the functions are likely implemented correctly'); +end +if FenchelYoungViolation + disp('Found violation of Fenchel-Young inequality; this is BAD'); +end +if break_on_bad + varargout{1} = []; + if nargout > 1, varargout{2} = []; end +else + varargout{1} = maxEr; +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/test_smooth.m b/test_smooth.m new file mode 100644 index 0000000..efac187 --- /dev/null +++ b/test_smooth.m @@ -0,0 +1,142 @@ +function varargout = test_smooth( f, N, DOM ) +% TEST_SMOOTH Runs diagnostic checks on a TFOCS smooth function object. +% TEST_SMOOTH( F ) +% tests whether the function handle F works as a TFOCS smooth +% function object. +% +% Requirements: f = F(X) must return the value of the function at X. +% [f,g] = F(X) must return the value, f, and the gradient, g. +% +% For an example, see SMOOTH_QUAD.M +% +% TEST_SMOOTH( F, N ) +% specifies the size of the domain space. If N is a scalar, +% then the domain space is the set of N x 1 vectors. +% If N is a matrix of the form [n1, n2], the the domain +% space is the set of n1 x n2 matrices. +% +% TEST_SMOOTH( ..., DOM ) +% specifies the domain of F. DOM can be of the form [a,b] +% which signifies the 1D set (a,b). +% +% OK = TEST_SMOOTH(...) +% returns "true" if all tests are passed, and "false" otherwise. +% +% See also private/tfocs_smooth, smooth_huber + + +OK = false; +PLOT = true; + +error(nargchk(1,3,nargin)); + +if ~isa(f,'function_handle') + fail('TFOCS smooth function must be a FUNCTION HANDLE'); + return; +end +fprintf('== Testing the smooth function %s ==\n', func2str(f) ); + +FORCE_N = false; % always require a vector input +if nargin < 2 || isempty(N), N = 10; +else FORCE_N = true; +end +if nargin < 3 || isempty(DOM), DOM = [-1,1]; end + +a = DOM(1); b = DOM(2); + +x = (a+b)/2; +fprintf('Testing scalar inputs... \n'); +try + vi = f(x); +catch + fail('TFOCS smooth function failed to return a function value'); + return; +end +if isinf(vi) + fail('TFOCS smooth function tester: default domain is invalid. Please specify valid domain'); + return; +end +fprintf('\t\t\t\t...passed. \n'); + +% Now, also ask for the gradient +fprintf('Testing gradient output... \n'); +try + [vi,gi] = f(x); +catch + fail('TFOCS smooth function failed to return a derivative value'); + return; +end +fprintf('\t\t\t\t...passed. \n'); + +% Now, try a vector +if isscalar(N) + x = repmat(x,N,1); +else + x = repmat(x,N(1),N(2)); + % if a > 0, assume we want a PSD matrix: + if a >= 0 + x = ones(N(1),N(2)) + eye(N(1),N(2) ); + end +end +fprintf('Testing vector inputs... \n'); +try + [vi,gi] = f(x); +catch + fail('TFOCS smooth function failed when supplied a vector input'); + return; +end +fprintf('\t\t\t\t...passed. \n'); + + + +% 1D example. Does not require function to be vectorized +n = 100; % number of grid points +h = (b-a)/n; +grid = (a+h/2):h:(b-h/2); + +v = zeros(size(grid)); +g = zeros(size(grid)); +first = @(x) x(1); +for i = 1:length(grid) + v(i) = f(grid(i)); + if isinf(v(i)) + g(i) = v(i); + else + [v(i),gi] = f(grid(i)); + g(i) = first(gi); + end +end + +if PLOT + figure; + clf; + plot(grid,v,'.-'); + hold all + plot(grid,g,'.-'); + legend('function','derivative'); +% title(func2str(f),'interpreter','none'); + line([a,b], 0*[1,1],'color','k' ) + if a < 0 && b > 0 + line( 0*[1,1], get(gca,'ylim'),'color','k' ); + end +end + +OK = true; + +if nargout > 0 + varargout{1} = OK; +end + +disp('Test passed succesfully.'); + + + + +function fail(str) + disp('Test failed. Reason:'); + disp(str); + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/tfocs.m b/tfocs.m new file mode 100644 index 0000000..b51f4a3 --- /dev/null +++ b/tfocs.m @@ -0,0 +1,112 @@ +function varargout = tfocs( smoothF, affineF, projectorF, x0, opts ) +% TFOCS Minimize a convex problem using a first-order algorithm. +% [ x, out, opts ] = tfocs( smoothF, affineF, projectorF, x0, opts ) +% Minimizes or maximizes a smooth or composite function using one of +% the TFOCS first-order algorithms. The nominal standard form that +% TFOCS handles is this: +% minimize smoothF(affineF(x))+projectorF(x) +% The affineF and projectorF functions are optional; specifically, they +% may be omitted by suppling an empty array [] in their place. Thus +% TFOCS supports standard forms such as +% minimize smoothF(affineF(x)) +% minimize smoothF(x) + projectorF(x) +% minimize smoothF(x) +% A variety of more advanced standard forms are supported, but only +% this simple form is discussed here. See the TFOCS user guide for +% more information on these advanced forms; see also TFOCS_SCD. +% +% FUNCTIONS: +% smoothF is expected to operate as follows: +% [ fx, gx ] = smoothF( x ) +% fx: the value of the smooth function at x +% gx: the gradient of the smooth function at x +% TFOCS only asks for the gradient if it needs it, so if they are +% significantly more expensive to compute, you should optimize +% your function for the 'nargout==1' case. +% +% affineF may take one of many forms: +% [] (empty): represents affineF(x) = x. +% A, where A is a matrix: represents affineF(x) = A*x. +% A, where A is a linear operator function obeying SPARCO conventions: +% y = A(x,0) or y = A([],0) returns the size of A. +% y = A(x,1) returns the forward operation applied to x. +% y = A(x,2) returns the adjoint operation applied to x. +% { A, b }, where A is either a matrix or linear operator and b +% is a matrix, adds the offset b to the forward operation. +% As mentioned above, affineF is optional. However, if the computational +% cost of evaluating your objective is dominated by an affine operation, +% it is worthwhile to provide it separately, as TFOCS orders its +% calculations to reduce the number of times it will be called. +% +% projectorF is the most complex, and has two modes of operation. +% Cx = projectorF( x ) +% Returns the value of the prox function at x. Note that no +% projection is taking place, so in many cases this should be +% a relatively simple calculation. +% [ Cz, z ] = projectorF( x, L ) +% Computes the minimizer +% z = argmin_z projectorF(z) + 0.5*L*\|x-y\|^2 +% The norm is Euclidean: \|z\| = ^{1/2}. +% +% OTHER INPUTS: +% x0: a feasible initial point +% opts: a structure containing further options. Please consult the TFOCS +% user guide for full details, but key entries include: +% opts.alg the algorithm to use; e.g., 'AT', 'LLM', etc. +% opts.maxIts max number of iterations +% opts.maxCounts max number of counts +% opts.tol tolerance for convergence: relative step length +% opts.printEvery displays output every 'printEvery' iteration +% opts.maxmin +1 to minimize (default), -1 to maximize +% Calling the function with no arguments displays the default values of +% opts, and returns that default structure in the third output. +% +% OUTPUTS +% x optimal point, up to the tolerance +% out contains extra information collected during the run +% opts structure containing the options used + +% User has requested viewing the default values of "opts" +if nargin == 0 || ( nargin==1 && isstruct(smoothF) ) + if nargout == 0 + tfocs_initialize; + else +% opts = tfocs_AT(); + tfocs_initialize; + opts.alg = 'AT'; + varargout{1} = opts; + end + return +elseif nargin == 1 && ischar(smoothF) && ... + (strcmpi(smoothF,'v') || strcmpi(smoothF,'version') ... + || strcmpi(smoothF,'-v') || strcmpi(smoothF,'-version') ) + % Display version information +% type version.txt + disp('TFOCS v1.3 1.1, December 2011'); + return +end + + +error(nargchk(1,5,nargin)); +if nargin < 2, affineF = []; end +if nargin < 3, projectorF = []; end +if nargin < 4, x0 = []; end +if nargin < 5, opts = []; end +if ~isfield( opts, 'alg' ), + alg = 'AT'; +else + alg = upper( opts.alg ); +end + +if isnumeric(affineF) && ~isempty(affineF) + affineF = {affineF}; +end + +[ varargout{1:max(nargout,1)} ] = feval( [ 'tfocs_', alg ], smoothF, affineF, projectorF, x0, opts ); +if nargout > 2, + varargout{3}.alg = alg; +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/tfocs_AT.m b/tfocs_AT.m new file mode 100644 index 0000000..0c30b1e --- /dev/null +++ b/tfocs_AT.m @@ -0,0 +1,92 @@ +function [ x, out, opts ] = tfocs_AT( smoothF, affineF, projectorF, x0, opts ) +% TFOCS_AT Auslender and Teboulle's accelerated method. +% [ x, out, opts ] = tfocs_AT( smoothF, affineF, nonsmoothF, x0, opts ) +% Implements Auslender & Teboulle's method. +% A variety of calling sequences are supported; type +% help tfocs_help +% for a full explanation. + +% Initialization +alg = 'AT'; +algorithm = 'Auslender & Teboulle''s single-projection method'; +alpha = 0; beta = 0; mu = 0; L = 0;% Do not remove: necessary because of a MATLAB quirk +tfocs_initialize +if nargin == 0, return; end + +while true, + + x_old = x; + A_x_old = A_x; + z_old = z; + A_z_old = A_z; + + % The backtracking loop + L_old = L; + L = L * alpha; + theta_old = theta; + while true, + + % Acceleration + theta = advance_theta( theta_old, L, L_old ); + + % Next iterate + if theta < 1, + y = ( 1 - theta ) * x_old + theta * z_old; + if cntr_Ay > cntr_reset + % every so often, we compute this explicitly, + % to avoid roundoff errors that might accumulate + A_y = apply_linear( y, 1 ); + cntr_Ay = 0; + else + % the efficient way + cntr_Ay = cntr_Ay + 1; + A_y = ( 1 - theta ) * A_x_old + theta * A_z_old; + end + f_y = Inf; g_Ay = []; g_y = []; + end + + % Compute the function value if it is not already + if isempty( g_y ), + if isempty( g_Ay ), [ f_y, g_Ay ] = apply_smooth( A_y ); end + g_y = apply_linear( g_Ay, 2 ); + end + + % Scaled gradient + step = 1 / ( theta * L ); + [ C_z, z ] = apply_projector( z_old - step * g_y, step ); + A_z = apply_linear( z, 1 ); + + % New iterate + if theta == 1, + x = z; + A_x = A_z; + C_x = C_z; + else + x = ( 1 - theta ) * x_old + theta * z; + if cntr_Ax > cntr_reset % see above comments for cntr_Ay + cntr_Ax = 0; + A_x = apply_linear( x, 1 ); + else + cntr_Ax = cntr_Ax + 1; + A_x = ( 1 - theta ) * A_x_old + theta * A_z; + end + C_x = Inf; + end + f_x = Inf; g_Ax = []; g_x = []; + + % Perform backtracking tests + tfocs_backtrack + + end + + % Collect data, evaluate stopping criteria, and print status + tfocs_iterate + +end + +% Final processing +tfocs_cleanup + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/tfocs_GRA.m b/tfocs_GRA.m new file mode 100644 index 0000000..1597fe5 --- /dev/null +++ b/tfocs_GRA.m @@ -0,0 +1,51 @@ +function [ x, out, opts ] = tfocs_GRA( smoothF, affineF, projectorF, x0, opts ) +%TFOCS_GRA Gradient descent. +% [ x, out, opts ] = tfocs_GRA( smoothF, affineF, nonsmoothF, x0, opts ) +% Implements a standard gradient method. A variety of calling sequences +% are supported; type 'help tfocs_help' for a full explanation. + +% Initialization +alg = 'GRA'; +algorithm = 'Proximal gradient descent'; +alpha = 0; beta = 0; mu = 0; L = 0; % Do not remove: necessary because of a MATLAB quirk +tfocs_initialize +if nargin==0, return; end + +% Unlike the other algorithms, GRA does not use the theta parameter, so it +% is not subject to restart, and we need not embed the initialization code +% inside the loop. + +while true, + + y = x; + A_y = A_x; + f_y = f_x; + x_old = x; + + g_Ay = g_Ax; + g_y = apply_linear( g_Ay, 2 ); + L = L * alpha; + + while true, + + % Standard gradient + [ C_x, x ] = apply_projector( y - (1/L) * g_y, 1/L ); + A_x = apply_linear( x, 1 ); + [ f_x, g_Ax ] = apply_smooth( A_x ); + + % Backtracking + tfocs_backtrack + + end + + % Collect data, evaluate stopping criteria, and print status + tfocs_iterate + +end + +% Final processing +tfocs_cleanup + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/tfocs_LLM.m b/tfocs_LLM.m new file mode 100644 index 0000000..939cf07 --- /dev/null +++ b/tfocs_LLM.m @@ -0,0 +1,77 @@ +function [ x, out, opts ] = tfocs_LLM( smoothF, affineF, projectorF, x0, opts ) +% TFOCS_LLM Lan, Lu and Monteiro's accelerated method. +% [ x, out, opts ] = tfocs_LLM( smoothF, affineF, nonsmoothF, x0, opts ) +% Implements Lan, Lu & Monteiro's method. +% A variety of calling sequences are supported; type +% help tfocs_help +% for a full explanation. + +% Initialization +alg = 'LLM'; +algorithm = 'Lan/Lu/Monteiro''s two-projection method'; +alpha = 0; beta = 0; mu =0; L = 0; % Do not remove: necessary due to MATLAB quirk +tfocs_initialize +if nargin == 0, return; end + + +while true, + + x_old = x; + A_x_old = A_x; + z_old = z; + A_z_old = A_z; + + % The backtracking loop + L_old = L; + L = L * alpha; + theta_old = theta; + while true, + + % Acceleration + theta = advance_theta( theta_old, L, L_old ); + + % Next iterate + if theta < 1, + y = ( 1 - theta ) * x_old + theta * z_old; + A_y = ( 1 - theta ) * A_x_old + theta * A_z_old; + f_y = Inf; g_Ay = []; g_y = []; + end + + % Compute the latest function values + if isempty( g_y ), + if isempty( g_Ay ), [ f_y, g_Ay ] = apply_smooth( A_y ); end + g_y = apply_linear( g_Ay, 2 ); + end + + % Standard gradient + [ C_x, x ] = apply_projector( y - (1/L) * g_y, 1/L ); + A_x = apply_linear( x, 1 ); + f_x = Inf; g_Ax = []; g_Ax = []; + + % Backtracking test + tfocs_backtrack + + end + + % Collect data, evaluate stopping criteria, and print status + tfocs_iterate + + % Scaled gradient. This step must be skipped if restart occurs + if theta == 1 || isinf(theta) + z = x; + C_z = C_x; + A_z = A_x; + else + step = 1 / ( theta * L ); + [ C_z, z ] = apply_projector( z - step * g_y, step ); + A_z = apply_linear( z, 1 ); + end + +end + +% Final processing +tfocs_cleanup + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/tfocs_N07.m b/tfocs_N07.m new file mode 100644 index 0000000..6f3e7e8 --- /dev/null +++ b/tfocs_N07.m @@ -0,0 +1,82 @@ +function [ x, out, opts ] = tfocs_N07( smoothF, affineF, projectorF, x0, opts ) +% TFOCS_N07 Nesterov's 2007 accelerated method. +% [ x, out, opts ] = tfocs_N07( smoothF, affineF, projectorF, x0, opts ) +% Implements Nesterov's 2007 two-projection method. +% A variety of calling sequences are supported; type +% help tfocs_help +% for a full explanation. + +% Initialization +alg = 'N07'; +algorithm = 'Nesterov''s 2007 two-projection method'; +alpha = 0; beta = 0; mu = 0; L = 0; % Necessary due to MATLAB quirk +tfocs_initialize +if nargin == 0,return; end + +while true, + + % Initialize the centerpoint and accumulated gradient. We do this here, + x_old = x; + A_x_old = A_x; + z_old = z; + A_z_old = A_z; + + % The backtracking loop + L_old = L; + L = L * alpha; + theta_old = theta; + while true, + + % Acceleration + theta = advance_theta( theta_old, L, L_old ); + + % Next iterate + if theta < 1, + y = ( 1 - theta ) * x_old + theta * z_old; + A_y = ( 1 - theta ) * A_x_old + theta * A_z_old; + f_y = Inf; g_Ay = []; g_y = []; + end + + % Compute function values + if isempty( g_y ), + if isempty( g_Ay ), + [ f_y, g_Ay ] = apply_smooth( A_y ); + end + g_y = apply_linear( g_Ay, 2 ); + end + + % Standard gradient + [ C_x, x ] = apply_projector( y - (1/L) * g_y, 1/L ); + A_x = apply_linear( x, 1 ); + f_x = Inf; g_Ax = []; g_x = []; + + % Backtracking test + tfocs_backtrack + + end + + % Collect data, evaluate stopping criteria, and print status + tfocs_iterate + + % Accumulated gradient. This step must be skipped if restart occurs + if theta == 1 || isinf(theta) + g_accum = g_y; + x_cent = x_old; + z = x; + C_z = C_x; + A_z = A_x; + else + g_accum = ( 1 - theta ) * g_accum + theta * g_y; + step = 1 / ( theta^2 * L ); + [ C_z, z ] = apply_projector( x_cent - step * g_accum, step ); + A_z = apply_linear( z, 1 ); + end + +end + +% Final processing +tfocs_cleanup + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/tfocs_N83.m b/tfocs_N83.m new file mode 100644 index 0000000..2c61825 --- /dev/null +++ b/tfocs_N83.m @@ -0,0 +1,78 @@ +function [ x, out, opts ] = tfocs_N83( smoothF, affineF, projectorF, x0, opts ) +% TFOCS_N83 Nesterov's 1983 accelerated method; also by Beck and Teboulle 2005 (FISTA). +% [ x, out, opts ] = tfocs_N83( smoothF, affineF, nonsmoothF, x0, opts ) +% Implements Nesterov's 1983 method. +% A variety of calling sequences are supported; type +% help tfocs_help +% for a full explanation. + +% Initialization +alg = 'N83'; +algorithm = 'Nesterov''s 1983 single-projection method'; +alpha = 0; beta = 0; mu = 0; L = 0; % Do not remove: necessary because of a MATLAB quirk +tfocs_initialize +if nargin == 0, return; end + +while true, + + x_old = x; + A_x_old = A_x; + z_old = z; + A_z_old = A_z; + + % The backtracking loop + L_old = L; + L = L * alpha; + theta_old = theta; + while true, + + % Acceleration + theta = advance_theta( theta_old, L, L_old ); + + % Next iterate + if theta < 1, + y = ( 1 - theta ) * x_old + theta * z_old; + A_y = ( 1 - theta ) * A_x_old + theta * A_z_old; + f_y = Inf; g_Ay = []; g_y = []; + end + + % Compute function values + if isempty( g_y ), + if isempty( g_Ay ), + [ f_y, g_Ay ] = apply_smooth( A_y ); + end + g_y = apply_linear( g_Ay, 2 ); + end + + % Standard gradient + [ C_x, x ] = apply_projector( y - (1/L) * g_y, 1/L ); + A_x = apply_linear( x, 1 ); + f_x = Inf; g_Ax = []; g_x = []; + + % Backtracking test + tfocs_backtrack + + end + + % Collect data, evaluate stopping criteria, and print status + tfocs_iterate + + % Reversed z update. This step is skipped if restart occurs + if theta == 1, + z = x; + A_z = A_x; + C_z = C_x; + else + z = (1/theta) * x + ( 1 - 1/theta ) * x_old; + A_z = (1/theta) * A_x + ( 1 - 1/theta ) * A_x_old; + C_z = Inf; + end + +end + +% Final processing +tfocs_cleanup + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/tfocs_SCD.m b/tfocs_SCD.m new file mode 100644 index 0000000..9d87bb2 --- /dev/null +++ b/tfocs_SCD.m @@ -0,0 +1,204 @@ +function [ x, odata, opts ] = tfocs_SCD( objectiveF, affineF, dualproxF, mu, x0, z0, opts, contOpts ) +% TFOCS_SCD Smoothed conic dual form of TFOCS, for problems with non-trivial linear operators. +% [ x, out, opts ] = tfocs_SCD( objectiveF, affineF, dualproxF, mu, x0, z0, opts ) +% Solves a conic problem using the smoothed conic dual approach. The goal +% is to solve a problem in the following conic form: +% minimize objectiveF(x)+0.5*mu(x-x0).^2 +% s.t. affineF(x) \in \cK +% The user is responsible for constructing the dual proximity function +% so that the dual can be described by the saddle point problem +% maximize_z inf_x [ objectiveF(x)+0.5*mu(x-x0).^2- ] -dualproxF(z) +% If mu = Inf, the method ignores the objective function and solves +% minimize 0.5*(x-x0).^2 +% s.t. affineF(x) \in \cK +% +% For general usage instructions, see tfocs.m and the user guide. +% In general, setting a variable to [] will set it to its default value. +% +% If opts.continuation is true, then +% [ x, out, opts ] = tfocs_SCD( objectiveF, affineF, dualproxF, mu, x0, z0, opts ) +% or +% [ x, out, opts ] = tfocs_SCD( objectiveF, affineF, dualproxF, mu, x0, z0, opts, continuationOptions ) +% will update x0 as to eliminate the influence of the mu(x-x0).^2 term +% Parameters for the continuation scheme are optionally specified +% in the "continuationOpts" variable. +% Note: if the "continuationOpts" variable is given, then the solver +% enters continuation mode automatically, unless opts.continuation +% is explicitly set to false. +% To see options available for continuation, type "continuation()" +% +% See also tfocs + +if nargin == 0 + % pass in the SCD specific options: + opSCD = []; + opSCD.continuation = false; + opSCD.adjoint = true; + opSCD.saddle = true; + opSCD.maxmin = -1; + if nargout > 0, x = tfocs(opSCD); + % and set the continuation options (right now, continuation is off by default) + x.continuation = false; + x = rmfield(x,'adjoint'); % x.adjoint = true; + x.saddle = true; + x.maxmin = -1; + else + tfocs(opSCD); + disp('Warning: we strongly recommend setting opts.continuation=true; see the user guide'); + disp('Type ''help continuation'' for details'); + end + return; +end + +error(nargchk(4,8,nargin)); +if nargin < 5, x0 = []; end +if nargin < 6, z0 = []; end +if nargin < 7, opts = []; end + +% look for continuation options +DO_CONTINUATION = false; +if nargin >= 8 + DO_CONTINUATION = true; +else + contOpts = []; +end +if isfield( opts, 'continuation' ) + if opts.continuation + DO_CONTINUATION = true; + else + DO_CONTINUATION = false; + end + opts = rmfield(opts,'continuation'); +end + +% Handle special cases of zero objective or infinite mu +if isinf( mu ), + mu = 1; + objectiveF = prox_0; +elseif isempty( objectiveF ), + objectiveF = prox_0; +elseif iscell( objectiveF ) % allow the case of {[],[],...,[]} + for k = 1:length(objectiveF) + if isempty( objectiveF{k} ) || isequal( objectiveF{k}, 0 ), + objectiveF{k} = prox_0; + elseif isnumeric( objectiveF{k} ), + objectiveF{k} = smooth_linear( objectiveF{k} ); + end + end +end + +% The affine quantities will be used in adjoint orientation +if isfield( opts, 'adjoint' ), + opts.adjoint = ~opts.adjoint; +else + opts.adjoint = true; +end +opts.saddle = true; +opts.maxmin = -1; +if isempty( x0 ), x0 = 0; end +smoothF = create_smoothF( objectiveF, mu, x0 ); + +if isempty(dualproxF) + dualproxF = proj_Rn; +elseif iscell(dualproxF) + for k = 1:length(dualproxF) + dp = dualproxF{k}; + if isempty( dp ) || isnumeric( dp ) && numel( dp ) == 1 && dp == 0, + dualproxF{k} = proj_Rn; + end + end +end +% When tfocs.m finds an error with z0, it says it has an error with "x0", +% which is confusing for tfocs_SCD users, since their "x0" has a different meaning. +% try % this is annoying for debugging + if DO_CONTINUATION + continuation_solver=@(mu,x0,z0,opts)solver(objectiveF,affineF,dualproxF, mu,x0,z0,opts); + [ x, odata, opts ] = continuation( continuation_solver, mu, x0, z0, opts,contOpts ); + else + [ z, odata, opts ] = tfocs( smoothF, affineF, dualproxF, z0, opts ); + end +% catch err +% if strfind( err.message, 'x0' ) +% fprintf(2,'Error involves z0 (which is referred to as x0 below)\n'); +% end +% rethrow(err); +% end +opts.adjoint = ~opts.adjoint; +opts = rmfield( opts, { 'saddle', 'maxmin' } ); +if DO_CONTINUATION + opts.continuation = true; +else + x = odata.dual; + odata.dual = z; +end + + + +% ----------------------- Subfunctions ---------------------------------------- + +function [ prox, x ] = smooth_dual( objectiveF, mu_i, x0, ATz ) +% Adding 0 to ATz will destroy the sparsity +if (isscalar(x0) && x0 == 0) || numel(x0) == 0 || nnz(x0) == 0 + [ v, x ] = objectiveF( mu_i * ATz, mu_i ); +else + [ v, x ] = objectiveF( x0 + mu_i * ATz, mu_i ); +end +prox = tfocs_dot( ATz, x ) - v - (0.5/mu_i) * tfocs_normsq( x - x0 ); +prox = -prox; +x = -x; + +function [ prox, x ] = smooth_dual_vectorMu( objectiveF, mu_i, x0, ATz ) +% Adding 0 to ATz will destroy the sparsity +if (isscalar(x0) && x0 == 0) || numel(x0) == 0 || nnz(x0) == 0 + [ v, x ] = objectiveF( mu_i .* ATz, mu_i ); +else + [ v, x ] = objectiveF( x0 + mu_i .* ATz, mu_i ); % Causes a scaling... +end +prox = tfocs_dot( ATz, x ) - v - 0.5*tfocs_normsq( x - x0, 1./mu_i ); +prox = -prox; +x = -x; + + +function smoothF = create_smoothF( objectiveF, mu, x0 ) +if iscell(objectiveF) + for k = 1 : length(objectiveF) + if iscell(x0) + if length(mu)>1 + smoothF{k} = @(varargin)smooth_dual_vectorMu( objectiveF{k}, 1./mu, x0{k}, varargin{:} ); + else + smoothF{k} = @(varargin)smooth_dual( objectiveF{k}, 1./mu, x0{k}, varargin{:} ); + end + else + if length(mu)>1 + smoothF{k} = @(varargin)smooth_dual_vectorMu( objectiveF{k}, 1./mu, x0, varargin{:} ); + else + smoothF{k} = @(varargin)smooth_dual( objectiveF{k}, 1./mu, x0, varargin{:} ); + end + end + end +else + if length(mu)>1 + smoothF = @(varargin)smooth_dual_vectorMu( objectiveF, 1./mu, x0, varargin{:} ); + else + smoothF = @(varargin)smooth_dual( objectiveF, 1./mu, x0, varargin{:} ); + end +end + +% For use with continuation: +function [varargout] = solver(objectiveF,affineF,dualproxF, mu,x0,z0,opts) +smoothF = create_smoothF( objectiveF, mu, x0 ); +[varargout{1:max(nargout,2)}] = tfocs( smoothF, affineF, dualproxF, z0, opts ); +% The varargout should be [x, odata, opts ] +% Since we're using the dual, need to switch +dualVar = varargout{1}; +odata = varargout{2}; +varargout{1} = odata.dual; +odata.dual = dualVar; +varargout{2} = odata; + + + + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/tfocs_TS.m b/tfocs_TS.m new file mode 100644 index 0000000..56734f7 --- /dev/null +++ b/tfocs_TS.m @@ -0,0 +1,86 @@ +function [ x, out, opts ] = tfocs_TS( smoothF, affineF, projectorF, x0, opts ) +% TFOCS_TS Tseng's modification of Nesterov's 2007 method. +% [ x, out, opts ] = tfocs_TS( smoothF, affineF, projectorF, x0, opts ) +% Implements Tseng's modification of the Nesterov 2007 method. +% A variety of calling sequences are supported; type +% help tfocs_help +% for a full explanation. + +% Initialization +alg = 'TS'; +algorithm = 'Tseng''s single-projection modification of Nesterov''s 2007 method'; +alpha = 0; beta = 0; mu = 0; L = 0; % Necessary due to MATLAB quirk +tfocs_initialize +if nargin == 0, return; end + +while true, + + x_old = x; + A_x_old = A_x; + z_old = z; + A_z_old = A_z; + + % The backtracking loop + L_old = L; + L = L * alpha; + theta_old = theta; + while true, + + % Acceleration + theta = 2 ./ ( 1 + sqrt( 1 + 4 * L / L_old / theta_old^2 ) ); + + % Next iterate + if theta < 1, + y = ( 1 - theta ) * x_old + theta * z_old; + A_y = ( 1 - theta ) * A_x_old + theta * A_z_old; + f_y = Inf; g_Ay = []; g_y = []; C_y = Inf; + end + + % Compute function values + if isempty( g_y ), + if isempty( g_Ay ), [ f_y, g_Ay ] = apply_smooth( A_y ); end + g_y = apply_linear( g_Ay, 2 ); + end + + % Accumulated gradient + if theta == 1, + x_cent = x_old; + g_a = g_y; + else + g_a = ( 1 - theta ) * g_a_old + theta * g_y; + end + step = 1 / ( theta^2 * L ); + [ C_z, z ] = apply_projector( x_cent - step * g_a, step ); + A_z = apply_linear( z, 1 ); + + % New iterate + if theta == 1, + x = z; + A_x = A_z; + C_x = C_z; + else + x = ( 1 - theta ) * x_old + theta * z; + A_x = ( 1 - theta ) * A_x_old + theta * A_z; + C_x = Inf; + end + f_x = Inf; g_Ax = []; g_x = []; + + % Bactracking test + tfocs_backtrack + + end + + % Collect data, evaluate stopping criteria, and print status + tfocs_iterate + + g_a_old = g_a; + +end + +% Final processing +tfocs_cleanup + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. + diff --git a/tfocs_normsq.m b/tfocs_normsq.m new file mode 100644 index 0000000..ebee4dd --- /dev/null +++ b/tfocs_normsq.m @@ -0,0 +1,22 @@ +function v = tfocs_normsq( x, scaling ) + +% TFOCS_NORMSQ Squared norm. +% By default, TFOCS_NORMSQ(X) = TFOCS_DOT(X,X). However, certain +% objects may have more efficient ways of computing this value. +% If so, TFOCS_NORMSQ should be overloaded to take advantage of +% this. However, the numerical equivalence to TFOCS_DOT(X,X) must +% be preserved. +% +% TFOCS_NORMSQ( X, D ) = TFOCS_DOT( X, D.*X ) is a scaled norm + +if nargin < 2 || isempty(scaling) + v = tfocs_dot( x, x ); +elseif numel(scaling) == 1 + v = scaling*tfocs_dot( x, x ); +else + v = tfocs_dot( x, scaling.*x ); +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/tfocs_version.m b/tfocs_version.m new file mode 100644 index 0000000..7996d1b --- /dev/null +++ b/tfocs_version.m @@ -0,0 +1,19 @@ +function tfocs_ver2 = tfocs_version + +%TFOCS_VERSION Version information. +% Prints information about the version of TFOCS being used, and the +% system on which it is running. When submitting a bug report, please +% run this function and include its output in your report. + +tfocs_ver = '1.3'; +if nargout == 0, + fprintf( 'TFOCS v%s\n', tfocs_ver ); + verd = ver('MATLAB'); + fprintf( 'MATLAB version %s %s on %s\n', verd.Version, verd.Release, computer ); +else + tfocs_ver2 = tfocs_ver; +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/tfocs_where.m b/tfocs_where.m new file mode 100644 index 0000000..b51279a --- /dev/null +++ b/tfocs_where.m @@ -0,0 +1,18 @@ +function s = tfocs_where + +%TFOCS_WHERE Returns the location of the TFOCS system. +% TFOCS_WHERE returns a string containing the base directory of the +% TFOCS solvers. Within that directory are some useful +% subdirectories and files: +% experiments/ sample TFOCS models +% COPYING.txt copyright information +% The proper operation of this function assumes that it has not been +% moved from its default position within the TFOCS distribution. + +s = mfilename('fullpath'); +temp = strfind( s, filesep ); +s( temp(end) + 1 : end ) = []; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/tfunc_scale.m b/tfunc_scale.m new file mode 100644 index 0000000..ec56364 --- /dev/null +++ b/tfunc_scale.m @@ -0,0 +1,95 @@ +function op = tfunc_scale( funcF, s, A, b ) + +%TFUNC_SCALE Scaling a function. +% SSCALE = TFUNC_SCALE( FUNC, s, A, b ) is the function +% SSCALE( y ) = s * FUNC( A * y + b ). +% s must be a real scalar; A can be a scalar, a matrix, or a linear +% operator; and offset must be a vector of compatible size. The third +% and fourth arguments are optional; A=I, b=0 are the defaults. If A +% is a matrix or a linear operator, then the resulting function cannot +% perform proximity minimizations. +% +% See also prox_scale + +error(nargchk(2,4,nargin)); +if ~isa( funcF, 'function_handle' ), + error( 'The first argument must be a function handle.' ); +elseif ~isnumeric( s ) || ~isreal( s ) || numel( s ) ~= 1, + error( 'The second argument must be a real scalar.' ); +end +if nargin < 3, + A = 1; +elseif ~isnumeric( A ) && ~isa( A, 'function_handle' ), + error( 'The third argument must be a scalar, matrix, or linear operator.' ); +end +if nargin < 4 || isempty( b ), + b = 0; +elseif ~isnumeric( b ), + error( 'The fourth argument must be a vector or matrix.' ); +end + +% Determine the best handle for the job +if s == 0, + op = smooth_constant( 0 ); +elseif ~isnumeric( A ), + op = @(varargin)tfunc_scale_linop( funcF, s, A, b, varargin{:} ); +elseif ~nnz(b) || numel( A ) ~= 1, + op = @(varargin)tfunc_scale_matrix( funcF, s, A, b, varargin{:} ); +elseif s ~= 1 || A ~= 1, + op = @(varargin)tfunc_scale_scalar( funcF, s, A, b, varargin{:} ); +else + op = funcF; +end + +function [ v, g ] = tfunc_scale_scalar( funcF, s, a, b, x, t ) +switch nargin, + case 5, + if nargout == 1, + v = funcF( a * x + b ); + else + [ v, g ] = funcF( a * x + b ); + g = ( s * A ) * g; + end + case 6, + [ v, g ] = funcF( a * x + b, s * a * t ); + g = ( g - b ) / a; + otherwise, + error( 'Not enough arguments.' ); +end +v = s * v; + +function [ v, g ] = tfunc_scale_matrix( funcF, s, A, b, x, t ) +switch nargin, + case 5, + if nargout == 1, + v = funcF( A * x + b ); + else + [ v, g ] = funcF( A * x + b ); + g = s * ( A' * g ); + end + case 6, + error( 'This function cannot perform proximity minimization.' ); + otherwise, + error( 'Not enough arguments.' ); +end +v = s * v; + +function [ v, g ] = tfunc_scale_linop( funcF, s, A, b, x, t ) +switch nargin, + case 5, + if nargout == 1, + v = funcF( A( x, 1 ) + b ); + else + [ v, g ] = funcF( A( x, 1 ) + b ); + g = s * A( g, 2 ); + end + case 6, + error( 'This function cannot perform proximity minimization.' ); + otherwise, + error( 'Not enough arguments.' ); +end +v = s * v; + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information. diff --git a/tfunc_sum.m b/tfunc_sum.m new file mode 100644 index 0000000..c720977 --- /dev/null +++ b/tfunc_sum.m @@ -0,0 +1,46 @@ +function op = tfunc_sum( varargin ) + +%TFUNC_SUM Sum of functions. +% OP = TFUNC_SUM( F1, F2, ..., FN ) implements +% OP( x ) = F1( x ) + F2( x ) + ... + FN( x ). +% Each entry must be a real scalar or a function handle. You are +% responsible for ensuring that the sum is convex or concave, as +% appropriate; TFOCS cannot verify this. + +for k = 1 : nargin, + arg = varargin{k}; + if ~isa( arg, 'function_handle' ) && ( ~isnumeric( arg ) || numel( arg ) ~= 1 || ~isreal( arg ) ), + error( 'Arguments must be function handles or real scalars.' ); + elseif isnumeric( arg ), + varargin{k} = smooth_constant( arg ); + end +end +switch nargin, + case 0, + op = smooth_constant( 0 ); + case 1, + op = varargin{1}; + otherwise, + op = @(x)tfunc_sum_impl( varargin, x ); +end + +function [ v, g ] = tfunc_sum_impl( args, x, t ) +if nargin > 2, + error( 'This function does not support proximity minimization.' ); +elseif nargout == 1, + v = args{1}( x ); + for k = 2 : numel(args), + v = v + args{k}( x ); + end +else + [ v, g ] = args{1}( x ); + for k = 2 : numel(args), + [ nv, ng ] = args{k}( x ); + v = v + nv; + g = g + ng; + end +end + +% TFOCS v1.3 by Stephen Becker, Emmanuel Candes, and Michael Grant. +% Copyright 2013 California Institute of Technology and CVX Research. +% See the file LICENSE for full license information.