Skip to content

Commit a86e135

Browse files
winmillwillwinmillwill
committed
Bundling up pending changes with experimental updates
1 parent 731fdc2 commit a86e135

13 files changed

+558
-36
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,4 @@ vendor/
1515
/.idea
1616
composer.phar
1717
/nbproject
18+
.vagrant

‎Berksfile

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
source 'https://api.berkshelf.com'
2+
3+
cookbook 'packagist', path: '~/chef/packagist'
4+
cookbook 'application_java', github: 'Youscribe/application_java-cookbook', branch: 'test'
5+
cookbook 'php-fpm', path: '~/chef/cookbook-php-fpm'
6+
cookbook 'application_solr', path: '~/chef/application_solr'
7+
cookbook 'nginx', '~>2.7'
8+
cookbook 'vagrant-ohai-plugin', github:'winmillwill/vagrant-ohai-plugin'

‎Berksfile.lock

+133
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
DEPENDENCIES
2+
application_java
3+
git: git://github.com/Youscribe/application_java-cookbook.git
4+
revision: 4f40f9b6e3cd6b5f8c2ba5fbd28c186670e820b8
5+
branch: test
6+
application_solr
7+
path: ../../chef/application_solr
8+
nginx (~> 2.7)
9+
packagist
10+
path: ../../chef/packagist
11+
php-fpm
12+
path: ../../chef/cookbook-php-fpm
13+
vagrant-ohai-plugin
14+
git: git://github.com/winmillwill/vagrant-ohai-plugin.git
15+
revision: f19d77a122376e21ea4e33c4bc89a6c99096f742
16+
17+
GRAPH
18+
7-zip (1.0.2)
19+
windows (>= 1.2.2)
20+
application (3.0.0)
21+
application_java (2.0.0)
22+
application (~> 3.0)
23+
java (>= 0.0.0)
24+
tomcat (>= 0.0.0)
25+
application_solr (0.1.0)
26+
application (>= 0.0.0)
27+
application_java (>= 0.0.0)
28+
apt (>= 0.0.0)
29+
ark (>= 0.0.0)
30+
apt (2.4.0)
31+
ark (0.9.0)
32+
7-zip (>= 0.0.0)
33+
windows (>= 0.0.0)
34+
bluepill (2.3.1)
35+
rsyslog (>= 0.0.0)
36+
build-essential (2.0.4)
37+
certificate (0.5.2)
38+
chef-sugar (1.3.0)
39+
chef_handler (1.1.6)
40+
composer (1.0.3)
41+
php (>= 0.0.0)
42+
windows (>= 0.0.0)
43+
discovery (0.2.2)
44+
dmg (2.2.0)
45+
erlang (1.5.4)
46+
apt (>= 1.7.0)
47+
build-essential (>= 0.0.0)
48+
yum (~> 3.0)
49+
yum-epel (>= 0.0.0)
50+
yum-erlang_solutions (>= 0.0.0)
51+
git (4.0.2)
52+
build-essential (>= 0.0.0)
53+
dmg (>= 0.0.0)
54+
runit (>= 1.0.0)
55+
windows (>= 0.0.0)
56+
yum (~> 3.0)
57+
yum-epel (>= 0.0.0)
58+
hostname (0.3.0)
59+
hostsfile (>= 0.0.0)
60+
hostsfile (2.4.4)
61+
iis (2.1.2)
62+
windows (>= 1.2.6)
63+
java (1.22.0)
64+
mysql (5.3.4)
65+
yum-mysql-community (>= 0.0.0)
66+
nginx (2.7.4)
67+
apt (~> 2.2)
68+
bluepill (~> 2.3)
69+
build-essential (~> 2.0)
70+
ohai (~> 2.0)
71+
runit (~> 1.2)
72+
yum-epel (~> 0.3)
73+
ohai (2.0.1)
74+
openssl (2.0.0)
75+
chef-sugar (>= 0.0.0)
76+
packagist (0.2.7)
77+
application_solr (>= 0.0.0)
78+
apt (>= 0.0.0)
79+
build-essential (>= 0.0.0)
80+
certificate (>= 0.0.0)
81+
composer (>= 0.0.0)
82+
discovery (>= 0.0.0)
83+
git (>= 0.0.0)
84+
hostname (>= 0.0.0)
85+
nginx (>= 0.0.0)
86+
php (>= 0.0.0)
87+
php-fpm (>= 0.0.0)
88+
rabbitmq (>= 0.0.0)
89+
redisio (>= 0.0.0)
90+
supervisor (>= 0.0.0)
91+
vagrant-ohai-plugin (>= 0.0.0)
92+
php (1.4.6)
93+
build-essential (>= 0.0.0)
94+
iis (>= 0.0.0)
95+
mysql (>= 0.0.0)
96+
windows (>= 0.0.0)
97+
xml (>= 0.0.0)
98+
yum-epel (>= 0.0.0)
99+
php-fpm (0.6.7)
100+
apt (>= 0.0.0)
101+
yum (>= 0.0.0)
102+
python (1.4.6)
103+
build-essential (>= 0.0.0)
104+
yum-epel (>= 0.0.0)
105+
rabbitmq (3.2.2)
106+
erlang (>= 0.9.0)
107+
redisio (1.7.1)
108+
ulimit (>= 0.1.2)
109+
rsyslog (1.12.2)
110+
runit (1.5.10)
111+
build-essential (>= 0.0.0)
112+
yum (~> 3.0)
113+
yum-epel (>= 0.0.0)
114+
supervisor (0.4.10)
115+
python (>= 0.0.0)
116+
tomcat (0.16.0)
117+
java (>= 0.0.0)
118+
openssl (>= 0.0.0)
119+
ulimit (0.3.2)
120+
vagrant-ohai-plugin (0.1.2)
121+
ohai (>= 0.0.0)
122+
windows (1.31.0)
123+
chef_handler (>= 0.0.0)
124+
xml (1.2.6)
125+
build-essential (>= 0.0.0)
126+
chef-sugar (>= 0.0.0)
127+
yum (3.2.2)
128+
yum-epel (0.3.6)
129+
yum (~> 3.0)
130+
yum-erlang_solutions (0.2.0)
131+
yum (~> 3.0)
132+
yum-mysql-community (0.1.6)
133+
yum (>= 3.0.0)

‎README.md

+38-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,42 @@
1-
Packagist
1+
(Drupal) Packagist
22
=========
33

4+
This is a hacked up fork of packagist for use with Drupal. The forking and
5+
hacking was done for want of a fast way to experiment with the problem domain
6+
while developing the Drupal-specific functionality separately.
7+
8+
All things considered, it would be best to provide separate repositories for the
9+
following:
10+
11+
* The Packagist/WebBundle by itself
12+
* HA functionality -- mostly the queueing used to bootstrap by traversing all
13+
the drupal.org project repos with worker nodes
14+
* Drupal-specific applications for the above
15+
* Drupal CLI tools for parsing update and release info as a thing unto itself
16+
17+
Instead, we have added the queuing and Drupal-specific functionality in place.
18+
The main workflow so far has been to install the application as normal and then
19+
populate the database so that you can generate a composer repository like so:
20+
21+
```
22+
./app/console packagist:bulk_add --repo-pattern \
23+
'http://git.drupal.org/project/%2$s' --vendor drupal $(curl \
24+
https://drupal.org/files/releases.tsv | grep 7.x | awk '{ print $3 }' | sort | uniq -)
25+
```
26+
27+
Running 10 AWS c3.large instances to consume the queue filled by the
28+
`packagist:bulk_add` command allows the process to complete in a few hours.
29+
30+
Experimental support for automatic updates has been added in the form of
31+
a foreground package upsert command that gets invoked by the
32+
`packagist:drupal_org_update` command, which parses the drupal 7 new releases
33+
rss feed. You would need to invoke this command with cron or similar to keep the
34+
application up to date with drupal.org and you would need to monitor disk space
35+
since the package information is read by cloning a bare repo from drupal.org and
36+
never removing it. You could consider updating the `drupal/parse_composer`
37+
project to add an appropriate cleanup method to the Repository class there, or
38+
just sweep out the cache directory composer uses at the end of the cron job.
39+
440
Package Repository Website for Composer, see the [about page](http://packagist.org/about) on [packagist.org](http://packagist.org/) for more.
541

642
Requirements
@@ -44,4 +80,4 @@ There are a few commands you should run periodically (ideally set up a cron job
4480
app/console packagist:dump --no-debug --env=prod
4581
app/console packagist:index --no-debug --env=prod
4682

47-
The latter is optional and only required if you are running a solr server.
83+
The latter is optional and only required if you are running a solr server.

‎Vagrantfile

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
Vagrant.configure('2') do |config|
2+
config.vm.box = 'precise64'
3+
config.vm.network :private_network, ip: '10.68.33.10'
4+
config.vm.provider :virtualbox do |vb|
5+
vb.customize ["modifyvm", :id, "--memory", "2048"]
6+
end
7+
config.vm.synced_folder '.', '/home/vagrant/packagist', :nfs => true
8+
9+
config.omnibus.chef_version = :latest
10+
config.berkshelf.enabled = true
11+
12+
config.vm.provision :chef_client do |chef|
13+
chef.chef_server_url = "https://chef.willmilton.com"
14+
chef.validation_key_path = "~/.chef/chef-validator.pem"
15+
chef.add_recipe 'build-essential'
16+
chef.add_role 'db_master'
17+
chef.add_recipe 'packagist::redis'
18+
chef.add_recipe 'packagist::solr'
19+
chef.add_recipe 'packagist::rabbitmq'
20+
chef.add_recipe 'packagist::bootstrap_app'
21+
chef.add_recipe 'packagist::app_worker'
22+
chef.json = {
23+
packagist: {
24+
web_root: '/var/www',
25+
ref: 'bg-updates',
26+
repository: 'https://github.com/winmillwill/packagist',
27+
nelmio_solarium: {
28+
clients: {
29+
default: {
30+
dsn: 'http://localhost:8080/solr/packagist'
31+
}
32+
}
33+
}
34+
},
35+
mysql: {
36+
server_root_password: 'password',
37+
server_repl_password: 'password',
38+
server_debian_password: 'password'
39+
}
40+
}
41+
end
42+
end

‎composer.lock

+22-18
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/Packagist/WebBundle/Command/DrupalOrgUpdatePackagesCommand.php

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
<?php
2+
3+
namespace Packagist\WebBundle\Command;
4+
5+
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
6+
use Symfony\Component\Console\Input\InputInterface;
7+
use Symfony\Component\Console\Input\InputOption;
8+
use Symfony\Component\Console\Input\InputArgument;
9+
use Symfony\Component\Console\Output\OutputInterface;
10+
use Packagist\WebBundle\Entity\Package;
11+
use Composer\Repository\VcsRepository;
12+
use Composer\IO\BufferIO;
13+
use Composer\IO\ConsoleIO;
14+
15+
class DrupalOrgUpdatePackagesCommand extends ContainerAwareCommand
16+
{
17+
protected function configure()
18+
{
19+
$this->setName('packagist:drupal_org_update')
20+
->setDescription('Updates packages with Drupal.org rss information');
21+
}
22+
23+
protected function execute(InputInterface $input, OutputInterface $output)
24+
{
25+
$parser = Factory::create();
26+
$parser->addFeed(
27+
'drupalOrg7x',
28+
'https://www.drupal.org/taxonomy/term/103/feed'
29+
);
30+
foreach ($parser->fetch('drupalOrg7x') as $item) {
31+
$update[current(explode(' ', $item->getName()))] = true;
32+
}
33+
$this->getApplication()->find('packagist:upsert')->run(
34+
new ArrayInput([
35+
'command' => 'packagist:upsert',
36+
'packages' => $update,
37+
'repo-pattern' => 'http://git.drupal.org/project/%2$s'
38+
]),
39+
$output
40+
);
41+
}
42+
}

0 commit comments

Comments
 (0)