fix for #967 filestore menu for all users

paulsputer · paulsputer · commit 697905cbf3b6 · 2015-12-07T23:09:26.000Z
+ Filestore listing filtered by user view permissions
+ Configuration help for filestore relocated to website files
+ Added migration example
diff --git a/HOME.md b/HOME.md
@@ -33,6 +33,8 @@ This documentation is the source content from which the [Gitblit website](http:/
 [[src/site/setup_viewer.mkd]]
 [[src/site/administration.mkd]]
 [[src/site/setup_scaling.mkd]]
+[[src/site/setup_filestore.mkd]]
+
 
 ### Gitblit Tickets
 
diff --git a/build.xml b/build.xml
@@ -514,6 +514,7 @@
 						<page name="mirrors" src="setup_mirrors.mkd" />
 						<page name="scaling" src="setup_scaling.mkd" />
 						<page name="fail2ban" src="setup_fail2ban.mkd" />
+						<page name="filestore (Git LFS)" src="setup_filestore.mkd" />
 						<divider />
 						<page name="Gitblit as a viewer" src="setup_viewer.mkd" />
 					</menu>
diff --git a/src/main/java/com/gitblit/manager/FilestoreManager.java b/src/main/java/com/gitblit/manager/FilestoreManager.java
@@ -77,6 +77,8 @@ public class FilestoreManager implements IFilestoreManager {
 	private final Logger logger = LoggerFactory.getLogger(getClass());
 
 	private final IRuntimeManager runtimeManager;
+	
+	private final IRepositoryManager repositoryManager;
 
 	private final IStoredSettings settings;
 
@@ -93,8 +95,10 @@ public class FilestoreManager implements IFilestoreManager {
 
 	@Inject
 	FilestoreManager(
-			IRuntimeManager runtimeManager) {
+			IRuntimeManager runtimeManager,
+			IRepositoryManager repositoryManager) {
 		this.runtimeManager = runtimeManager;
+		this.repositoryManager = repositoryManager;
 		this.settings = runtimeManager.getSettings();
 	}
 
@@ -324,8 +328,29 @@ public FilestoreModel.Status downloadBlob(String oid, UserModel user, Repository
 	}
 
 	@Override
-	public List<FilestoreModel> getAllObjects() {
-		return new ArrayList<FilestoreModel>(fileCache.values());
+	public List<FilestoreModel> getAllObjects(UserModel user) {
+		
+		final List<RepositoryModel> viewableRepositories = repositoryManager.getRepositoryModels(user);
+		List<String> viewableRepositoryNames = new ArrayList<String>(viewableRepositories.size());
+		
+		for (RepositoryModel repository : viewableRepositories) {
+			viewableRepositoryNames.add(repository.name);
+		}
+		
+		if (viewableRepositoryNames.size() == 0) {
+			return null;
+		}
+		
+		final Collection<FilestoreModel> allFiles = fileCache.values();
+		List<FilestoreModel> userViewableFiles = new ArrayList<FilestoreModel>(allFiles.size());
+		
+		for (FilestoreModel file : allFiles) {
+			if (file.isInRepositoryList(viewableRepositoryNames)) {
+				userViewableFiles.add(file);
+			}
+		}
+		
+		return userViewableFiles;				
 	}
 
 	@Override
diff --git a/src/main/java/com/gitblit/manager/GitblitManager.java b/src/main/java/com/gitblit/manager/GitblitManager.java
@@ -1274,8 +1274,8 @@ public FilestoreModel.Status downloadBlob(String oid, UserModel user, Repository
 	}
 	
 	@Override
-	public List<FilestoreModel> getAllObjects() {
-		return filestoreManager.getAllObjects();
+	public List<FilestoreModel> getAllObjects(UserModel user) {
+		return filestoreManager.getAllObjects(user);
 	}
 	
 	@Override
diff --git a/src/main/java/com/gitblit/manager/IFilestoreManager.java b/src/main/java/com/gitblit/manager/IFilestoreManager.java
@@ -37,7 +37,7 @@ public interface IFilestoreManager extends IManager {
 	
 	FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut );
 	
-	List<FilestoreModel> getAllObjects();
+	List<FilestoreModel> getAllObjects(UserModel user);
 	
 	File getStorageFolder();
 	
diff --git a/src/main/java/com/gitblit/models/FilestoreModel.java b/src/main/java/com/gitblit/models/FilestoreModel.java
@@ -111,6 +111,15 @@ public synchronized void removeRepository(String repo) {
 		repositories.remove(repo);
 	}
 	
+	public synchronized boolean isInRepositoryList(List<String> repoList) {
+		for (String name : repositories) {
+			if (repoList.contains(name)) {
+				return true;
+			}
+		}
+		return false;
+	}
+	
 	public static enum Status {
 
 		Deleted(-30),
diff --git a/src/main/java/com/gitblit/wicket/pages/FilestorePage.java b/src/main/java/com/gitblit/wicket/pages/FilestorePage.java
@@ -18,6 +18,7 @@
 import java.text.DateFormat;
 import java.text.MessageFormat;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.io.FileUtils;
@@ -31,27 +32,35 @@
 import com.gitblit.Constants;
 import com.gitblit.models.FilestoreModel;
 import com.gitblit.models.UserModel;
+import com.gitblit.wicket.CacheControl;
 import com.gitblit.wicket.FilestoreUI;
+import com.gitblit.wicket.GitBlitWebSession;
 import com.gitblit.wicket.RequiresAdminRole;
 import com.gitblit.wicket.WicketUtils;
+import com.gitblit.wicket.CacheControl.LastModified;
 
 /**
  * Page to display the current status of the filestore.
  * Certain errors also displayed to aid in fault finding
  *
  * @author Paul Martin
  */
-@RequiresAdminRole
+@CacheControl(LastModified.ACTIVITY)
 public class FilestorePage extends RootPage {
 
 	public FilestorePage() {
 		super();
 		setupPage("", "");
 
-		final List<FilestoreModel> files = app().filestore().getAllObjects();
+		final UserModel user = (GitBlitWebSession.get().getUser() == null) ? UserModel.ANONYMOUS : GitBlitWebSession.get().getUser();
 		final long nBytesUsed = app().filestore().getFilestoreUsedByteCount();
 		final long nBytesAvailable = app().filestore().getFilestoreAvailableByteCount();
+		List<FilestoreModel> files = app().filestore().getAllObjects(user);
 
+		if (files == null) {
+			files = new ArrayList<FilestoreModel>();
+		}
+		
 		String message = MessageFormat.format(getString("gb.filestoreStats"), files.size(),
 				FileUtils.byteCountToDisplaySize(nBytesUsed), FileUtils.byteCountToDisplaySize(nBytesAvailable) );
 
@@ -64,7 +73,6 @@ public FilestorePage() {
 		helpLink.add(new Label("helpMessage", getString("gb.filestoreHelp")));
 		add(helpLink);
 
-
 		DataView<FilestoreModel> filesView = new DataView<FilestoreModel>("fileRow",
 				new ListDataProvider<FilestoreModel>(files)) {
 			private static final long serialVersionUID = 1L;
diff --git a/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html
@@ -12,16 +12,19 @@
 <div class="span10 offset1">	
 	
 	<div class="alert alert-danger">
-	<h3><center>Using the Filestore</center></h3>
+	<h3><center>Using the filestore</center></h3>
 	<p>
-		<strong>All clients intending to use the filestore must first install the <a href="https://git-lfs.github.com/">Git-LFS Client</a> and then run <code>git lfs init</code> to register the hooks globally.</strong><br/>
-		<i>This version of GitBlit has been verified with Git-LFS client version 0.6.0 which requires Git v1.8.2 or higher.</i>
+		<strong>All clients intending to use the filestore must first install the <a href="https://git-lfs.github.com/">Git-LFS Client</a> and then run <code>git lfs install</code></strong><br/>
+		<p>
+		If using password authentication it is recommended that you configure the <a href="https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage">git credential storage</a> to avoid Git-LFS asking for your password on each file<br/>
+		On Windows for example: <code>git config --global credential.helper wincred</code>
+		</p>
 	</p>
 	</div>
 		
 	<h3>Clone</h3>
 	<p>
-	Just <code>git clone</code> as usual, no further action is required as GitBlit is configured to use the default Git-LFS end point <code>{repository}/info/lfs/objects/</code>.<br/>
+	Just <code>git clone</code> as usual, no further action is required as Gitblit is configured to use the default Git-LFS end point <code>{repository}/info/lfs/objects/</code>.<br/>
 	<i>If the repository uses a 3rd party Git-LFS server you will need to <a href="https://github.com/github/git-lfs/blob/master/docs/spec.md#the-server">manually configure the correct endpoints</a></i>.
 	</p>
 	
@@ -38,28 +41,7 @@ <h3>Learn more...</h3>
 	<p><a href="https://github.com/github/git-lfs/blob/master/docs/spec.md">See the current Git-LFS specification for further details</a>.</p>
 	<br />
 	
-	<div class="alert alert-warn">
-	<h3><center>Limitations & Warnings</center></h3>
-	<p>GitBlit currently provides a server-only implementation of the opensource Git-LFS API, <a href="https://github.com/github/git-lfs/wiki/Implementations">other implementations</a> are available.<br/>
-	However, until <a href="https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333">JGit provides Git-LFS client capabilities</a> some GitBlit features may not be fully supported when using the filestore.
-	Notably:
-		<ul>
-		<li>Mirroring a repository that uses Git-LFS - Only the pointer files, not the large files, are mirrored.</li>
-		<li>Federation -  Only the pointer files, not the large files, are transfered.</li>
-		</ul>
-	</p>
-	</div>
-	
-	<div class="alert alert-info">
-	<h3><center>GitBlit Configuration</center></h3>
-	<p>GitBlit provides the following configuration items when using the filestore:
-		<h4>filestore.storageFolder</h4>
-		<p>Defines the path on the server where filestore objects are to be saved. This defaults to <code>${baseFolder}/lfs</code></p>
-		<h4>filestore.maxUploadSize</h4>
-		<p>Defines the maximum allowable size that can be uploaded to the filestore.  Once a file is uploaded it will be unaffected by later changes in this property. This defaults to <code>-1</code> indicating no limits.</p>
-	</p>
-	</div>
-	
+
 </div>
 </div>
 </div>
diff --git a/src/main/java/com/gitblit/wicket/pages/RootPage.java b/src/main/java/com/gitblit/wicket/pages/RootPage.java
@@ -197,9 +197,9 @@ public void renderHead(IHeaderResponse response) {
 			}
 			navLinks.add(new PageNavLink("gb.repositories", RepositoriesPage.class,
 					getRootPageParameters()));
-			if (user.canAdmin()) {
-				navLinks.add(new PageNavLink("gb.filestore", FilestorePage.class, getRootPageParameters()));
-			}
+			
+			navLinks.add(new PageNavLink("gb.filestore", FilestorePage.class, getRootPageParameters()));
+				
 			navLinks.add(new PageNavLink("gb.activity", ActivityPage.class, getRootPageParameters()));
 			if (allowLucene) {
 				navLinks.add(new PageNavLink("gb.search", LuceneSearchPage.class));
diff --git a/src/site/setup_filestore.mkd b/src/site/setup_filestore.mkd
@@ -0,0 +1,61 @@
+## Configure Git Large File Storage
+
+Gitblit provides a filestore that supports the [Git Large File Storage (LFS) API](https://git-lfs.github.com/).
+
+### Server Configuration
+
+Gitblit is configured to work straight away.  However you may want to update the following in `gitblit.properties`:
+
+<table class="table">
+<thead>
+<tr><th>parameter</th><th>value</th><th>description</th></tr>
+</thead>
+<tbody>
+<tr>
+  <th>filestore.storageFolder</th><td>${baseFolder}/lfs</td>
+  <td>The path on the server where filestore objects are to be saved.</td>
+</tr>
+<tr>
+  <th>filestore.maxUploadSize</th><td>-1</td>
+  <td>The maximum allowable size that can be uploaded to the filestore.  Once a file is uploaded it will be unaffected by later changes in this property. The default of -1 indicates no limits.</td>
+</tr>
+</tbody>
+</table>
+
+	
+### Limitations
+
+Gitblit currently provides a server-only implementation of the opensource Git LFS API.
+
+1. Files in the filestore are not currently searchable by Lucene.
+2. Mirroring a repository that uses Git LFS will only mirror the pointer files, not the large files.
+3. Federation - Only the pointer files, not the large files, are transfered.
+
+Items 2 & 3 are pending [JGit Git LFS client capabilities](https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333).
+
+
+### How does it work?
+
+1. Files that should be handled by Git LFS are defined in the `.gitattributes` file.
+2. Git LFS installs a pre-commit hook when installed `git lfs install`.
+3. When a commit is made the pre-commit hook replaces the defined Git LFS files with a pointer file containing metadata about the file so that it can be found later.
+4. When a commit is pushed, the changeset is sent to the git repository and the large files are sent to the filestore.   
+
+For further details check out the [Git LFS specification](https://github.com/github/git-lfs/blob/master/docs/spec.md).
+
+### Convert/Migrate existing repository
+
+It is possible to migrate an existing repository containing large files to one that leverages the filestore.  However, commit hash history will be altered.
+
+The following command may be run on a local repository:
+
+    git filter-branch --prune-empty --tree-filter '
+    git lfs track "*.docx" "*.pdf" > /dev/null
+    git add .gitattributes
+    git ls-files | xargs -d "\n" git check-attr filter | grep "filter: lfs" | sed -r "s/(.*): filter: lfs/\1/" | xargs -d "\n" -r bash -c "git rm -f --cached \"\$@\"; git add \"\$@\"" bash \
+    ' --tag-name-filter cat -- --all
+
+
+### Further Considerations
+
+While [other Git LFS implementations are available](https://github.com/github/git-lfs/wiki/Implementations) as there is no current [JGit LFS client capability](https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333), Gitblit will be unable to access them.   

Original file line number	Diff line number	Diff line change
`@@ -1274,8 +1274,8 @@ public FilestoreModel.Status downloadBlob(String oid, UserModel user, Repository`
`1274`	`1274`	`}`
`1275`	`1275`
`1276`	`1276`	`@Override`
`1277`		`- public List<FilestoreModel> getAllObjects() {`
`1278`		`- return filestoreManager.getAllObjects();`
	`1277`	`+ public List<FilestoreModel> getAllObjects(UserModel user) {`
	`1278`	`+ return filestoreManager.getAllObjects(user);`
`1279`	`1279`	`}`
`1280`	`1280`
`1281`	`1281`	`@Override`