Skip to content

Commit

Permalink
Allow log root directory to be specified via command line flag. (#627)
Browse files Browse the repository at this point in the history
  • Loading branch information
@bcipriano
bcipriano authored Feb 21, 2020
1 parent 5b5bd56 commit 4b7b789
Show file tree
Hide file tree
Showing 13 changed files with 86 additions and 55 deletions.
3 changes: 2 additions & 1 deletion cuebot/src/main/java/com/imageworks/spcue/dao/JobDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import com.imageworks.spcue.ShowInterface;
import com.imageworks.spcue.TaskEntity;
import com.imageworks.spcue.grpc.job.JobState;
import com.imageworks.spcue.util.JobLogUtil;

public interface JobDao {

Expand Down Expand Up @@ -115,7 +116,7 @@ public interface JobDao {
*
* @param j
*/
void insertJob(JobDetail j);
void insertJob(JobDetail j, JobLogUtil jobLogUtil);

/**
* Finds a Job from its name. This method returns only
Expand Down
4 changes: 2 additions & 2 deletions cuebot/src/main/java/com/imageworks/spcue/dao/oracle/JobDaoJdbc.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -441,9 +441,9 @@ public boolean updateJobFinished(JobInterface job) {
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";

@Override
public void insertJob(JobDetail j) {
public void insertJob(JobDetail j, JobLogUtil jobLogUtil) {
j.id = SqlUtil.genKeyRandom();
j.logDir = JobLogUtil.getJobLogPath(j);
j.logDir = jobLogUtil.getJobLogPath(j);
if (j.minCoreUnits < 100) { j.minCoreUnits = 100; }

getJdbcTemplate().update(INSERT_JOB,
Expand Down
4 changes: 2 additions & 2 deletions cuebot/src/main/java/com/imageworks/spcue/dao/postgres/JobDaoJdbc.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -439,9 +439,9 @@ public boolean updateJobFinished(JobInterface job) {
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";

@Override
public void insertJob(JobDetail j) {
public void insertJob(JobDetail j, JobLogUtil jobLogUtil) {
j.id = SqlUtil.genKeyRandom();
j.logDir = JobLogUtil.getJobLogPath(j);
j.logDir = jobLogUtil.getJobLogPath(j);
if (j.minCoreUnits < 100) { j.minCoreUnits = 100; }

getJdbcTemplate().update(INSERT_JOB,
Expand Down
13 changes: 11 additions & 2 deletions cuebot/src/main/java/com/imageworks/spcue/service/JobManagerService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ public class JobManagerService implements JobManager {
private FilterManager filterManager;
private GroupDao groupDao;
private FacilityDao facilityDao;
private JobLogUtil jobLogUtil;

@Transactional(propagation = Propagation.REQUIRED, readOnly=true)
public boolean isJobComplete(JobInterface job) {
Expand Down Expand Up @@ -242,7 +243,7 @@ public JobDetail createJob(BuildableJob buildableJob) {

resolveFacility(job);

jobDao.insertJob(job);
jobDao.insertJob(job, jobLogUtil);
jobDao.insertEnvironment(job, buildableJob.env);

for (BuildableLayer buildableLayer: buildableJob.getBuildableLayers()) {
Expand Down Expand Up @@ -366,7 +367,7 @@ public void markFrameAsDepend(FrameInterface frame) {
*/
@Transactional(propagation = Propagation.NEVER)
public void createJobLogDirectory(JobDetail job) {
if (!JobLogUtil.createJobLogDirectory(job.logDir)) {
if (!jobLogUtil.createJobLogDirectory(job.logDir)) {
throw new JobLaunchException("error launching job, unable to create log directory");
}
}
Expand Down Expand Up @@ -623,5 +624,13 @@ public HostDao getHostDao() {
public void setHostDao(HostDao hostDao) {
this.hostDao = hostDao;
}

public JobLogUtil getJobLogUtil() {
return jobLogUtil;
}

public void setJobLogUtil(JobLogUtil jobLogUtil) {
this.jobLogUtil = jobLogUtil;
}
}

37 changes: 12 additions & 25 deletions cuebot/src/main/java/com/imageworks/spcue/util/JobLogUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,39 +16,31 @@
*/



package com.imageworks.spcue.util;

import java.io.File;

import org.springframework.beans.factory.annotation.Value;
import com.imageworks.spcue.JobDetail;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;

import com.imageworks.spcue.JobDetail;
import java.io.File;

@Component
public class JobLogUtil {

public static String jobLogRootDir;
@Autowired
private Environment env;

public static boolean createJobLogDirectory(String path) {
public boolean createJobLogDirectory(String path) {
File f = new File(path);
f.mkdir();
f.setWritable(true, false);
return f.isDirectory();
}

public static boolean shotLogDirectoryExists(String show, String shot) {
return new File(getJobLogDir(show, shot)).exists();
}

public static boolean jobLogDirectoryExists(JobDetail job) {
return new File(job.logDir).exists();
}

public static String getJobLogDir(String show, String shot) {
public String getJobLogDir(String show, String shot) {
StringBuilder sb = new StringBuilder(512);
sb.append(jobLogRootDir);
sb.append(getJobLogRootDir());
sb.append("/");
sb.append(show);
sb.append("/");
Expand All @@ -57,7 +49,7 @@ public static String getJobLogDir(String show, String shot) {
return sb.toString();
}

public static String getJobLogPath(JobDetail job) {
public String getJobLogPath(JobDetail job) {
StringBuilder sb = new StringBuilder(512);
sb.append(getJobLogDir(job.showName, job.shot));
sb.append("/");
Expand All @@ -67,13 +59,8 @@ public static String getJobLogPath(JobDetail job) {
return sb.toString();
}

public static String getJobLogRootDir() {
return jobLogRootDir;
}

@Value("${log.frameLogDirRoot}")
public void setJobLogRootDir(String logRootDir) {
jobLogRootDir = logRootDir;
public String getJobLogRootDir() {
return env.getRequiredProperty("log.frame-log-root", String.class);
}
}

3 changes: 3 additions & 0 deletions cuebot/src/main/resources/conf/spring/applicationContext-service.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,8 @@
<property name="jobDao" ref="jobDao" />
</bean>

<bean id="jobLogUtil" class="com.imageworks.spcue.util.JobLogUtil" />

<bean id="jobManager" class="com.imageworks.spcue.service.JobManagerService">
<property name="jobDao" ref="jobDao" />
<property name="showDao" ref="showDao" />
Expand All @@ -170,6 +172,7 @@
<property name="filterManager" ref="filterManager" />
<property name="hostDao" ref="hostDao" />
<property name="limitDao" ref="limitDao" />
<property name="jobLogUtil" ref="jobLogUtil" />
</bean>

<bean id="mailSender" class="org.springframework.mail.javamail.JavaMailSenderImpl">
Expand Down
4 changes: 3 additions & 1 deletion cuebot/src/main/resources/opencue.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ grpc.rqd_cache_expiration=30
messaging.enabled=false

# Root directory for which logs will be stored. See com/imageworks/spcue/util/JobLogUtil.java.
log.frameLogDirRoot=${CUE_FRAME_LOG_DIR:/shots}
# Override this via environment variable (CUE_FRAME_LOG_DIR) or command line flag
# (--log.frame-log-root). Command line flag will be preferred if both are provided.
log.frame-log-root=${CUE_FRAME_LOG_DIR:/shots}

# Maximum number of jobs to query.
dispatcher.job_query_max=20
Expand Down
11 changes: 7 additions & 4 deletions cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/JobDaoTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,9 @@ public class JobDaoTests extends AbstractTransactionalJUnit4SpringContextTests
@Resource
DepartmentDao departmentDao;

@Resource
JobLogUtil jobLogUtil;

private static String ROOT_FOLDER = "A0000000-0000-0000-0000-000000000000";
private static String ROOT_SHOW = "00000000-0000-0000-0000-000000000000";
private static String JOB_NAME = "pipe-dev.cue-testuser_shell_v1";
Expand All @@ -117,11 +120,11 @@ public JobDetail insertJob() {
JobDetail job = this.buildJobDetail();
job.groupId = ROOT_FOLDER;
job.showId = ROOT_SHOW;
job.logDir = JobLogUtil.getJobLogPath(job);
job.logDir = jobLogUtil.getJobLogPath(job);
job.deptId = departmentDao.getDefaultDepartment().getId();
job.facilityId = facilityDao.getDefaultFacility().getId();
job.state = JobState.PENDING;
jobDao.insertJob(job);
jobDao.insertJob(job, jobLogUtil);
return job;
}

Expand Down Expand Up @@ -156,10 +159,10 @@ public void testInsertJob() {
JobDetail job = this.buildJobDetail();
job.groupId = ROOT_FOLDER;
job.showId = ROOT_SHOW;
job.logDir = JobLogUtil.getJobLogPath(job);
job.logDir = jobLogUtil.getJobLogPath(job);
job.deptId = departmentDao.getDefaultDepartment().getId();
job.facilityId= facilityDao.getDefaultFacility().getId();
jobDao.insertJob(job);
jobDao.insertJob(job, jobLogUtil);
assertNotNull(job.id);
}

Expand Down
7 changes: 5 additions & 2 deletions cuebot/src/test/java/com/imageworks/spcue/test/dao/oracle/LayerDaoTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,9 @@ public class LayerDaoTests extends AbstractTransactionalJUnit4SpringContextTests
@Resource
FacilityDao facilityDao;

@Resource
JobLogUtil jobLogUtil;

private static String ROOT_FOLDER = "A0000000-0000-0000-0000-000000000000";
private static String ROOT_SHOW = "00000000-0000-0000-0000-000000000000";
private static String LAYER_NAME = "pass_1";
Expand All @@ -116,10 +119,10 @@ public LayerDetail getLayer() {
JobDetail job = spec.getJobs().get(0).detail;
job.groupId = ROOT_FOLDER;
job.showId = ROOT_SHOW;
job.logDir = JobLogUtil.getJobLogPath(job);
job.logDir = jobLogUtil.getJobLogPath(job);
job.deptId = departmentDao.getDefaultDepartment().getId();
job.facilityId = facilityDao.getDefaultFacility().getId();
jobDao.insertJob(job);
jobDao.insertJob(job, jobLogUtil);

LayerDetail lastLayer= null;
String limitId = limitDao.createLimit(LIMIT_NAME, LIMIT_MAX_VALUE);
Expand Down
11 changes: 7 additions & 4 deletions cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/JobDaoTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,9 @@ public class JobDaoTests extends AbstractTransactionalJUnit4SpringContextTests
@Resource
DepartmentDao departmentDao;

@Resource
JobLogUtil jobLogUtil;

private static String ROOT_FOLDER = "A0000000-0000-0000-0000-000000000000";
private static String ROOT_SHOW = "00000000-0000-0000-0000-000000000000";
private static String JOB_NAME = "pipe-dev.cue-testuser_shell_v1";
Expand All @@ -117,11 +120,11 @@ public JobDetail insertJob() {
JobDetail job = this.buildJobDetail();
job.groupId = ROOT_FOLDER;
job.showId = ROOT_SHOW;
job.logDir = JobLogUtil.getJobLogPath(job);
job.logDir = jobLogUtil.getJobLogPath(job);
job.deptId = departmentDao.getDefaultDepartment().getId();
job.facilityId = facilityDao.getDefaultFacility().getId();
job.state = JobState.PENDING;
jobDao.insertJob(job);
jobDao.insertJob(job, jobLogUtil);
return job;
}

Expand Down Expand Up @@ -156,10 +159,10 @@ public void testInsertJob() {
JobDetail job = this.buildJobDetail();
job.groupId = ROOT_FOLDER;
job.showId = ROOT_SHOW;
job.logDir = JobLogUtil.getJobLogPath(job);
job.logDir = jobLogUtil.getJobLogPath(job);
job.deptId = departmentDao.getDefaultDepartment().getId();
job.facilityId= facilityDao.getDefaultFacility().getId();
jobDao.insertJob(job);
jobDao.insertJob(job, jobLogUtil);
assertNotNull(job.id);
}

Expand Down
8 changes: 5 additions & 3 deletions cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/LayerDaoTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,13 +89,15 @@ public class LayerDaoTests extends AbstractTransactionalJUnit4SpringContextTests
@Resource
JobLauncher jobLauncher;


@Resource
DepartmentDao departmentDao;

@Resource
FacilityDao facilityDao;

@Resource
JobLogUtil jobLogUtil;

private static String ROOT_FOLDER = "A0000000-0000-0000-0000-000000000000";
private static String ROOT_SHOW = "00000000-0000-0000-0000-000000000000";
private static String LAYER_NAME = "pass_1";
Expand All @@ -117,10 +119,10 @@ public LayerDetail getLayer() {
JobDetail job = spec.getJobs().get(0).detail;
job.groupId = ROOT_FOLDER;
job.showId = ROOT_SHOW;
job.logDir = JobLogUtil.getJobLogPath(job);
job.logDir = jobLogUtil.getJobLogPath(job);
job.deptId = departmentDao.getDefaultDepartment().getId();
job.facilityId = facilityDao.getDefaultFacility().getId();
jobDao.insertJob(job);
jobDao.insertJob(job, jobLogUtil);

LayerDetail lastLayer= null;
String limitId = limitDao.createLimit(LIMIT_NAME, LIMIT_MAX_VALUE);
Expand Down
34 changes: 25 additions & 9 deletions cuebot/src/test/java/com/imageworks/spcue/test/util/JobLogUtilTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,37 +15,53 @@
*/



package com.imageworks.spcue.test.util;

import junit.framework.TestCase;
import org.junit.Before;

import com.imageworks.spcue.JobDetail;
import com.imageworks.spcue.config.TestAppConfig;
import com.imageworks.spcue.util.JobLogUtil;
import org.junit.Before;
import org.junit.Test;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.AbstractTransactionalJUnit4SpringContextTests;
import org.springframework.test.context.support.AnnotationConfigContextLoader;

import javax.annotation.Resource;

import static org.junit.Assert.assertEquals;

@ContextConfiguration(classes = TestAppConfig.class, loader = AnnotationConfigContextLoader.class)
public class JobLogUtilTests extends AbstractTransactionalJUnit4SpringContextTests {

@Resource
private JobLogUtil jobLogUtil;

public class JobLogUtilTests extends TestCase {
private String logRoot;

@Before
public void setUp() {
JobLogUtil.jobLogRootDir = "/shots";
// This value should match what's defined in test/resources/opencue.properties.
logRoot = "/arbitraryLogDirectory";
}

@Test
public void testGetJobLogRootDir() {
assertEquals(JobLogUtil.getJobLogRootDir(), "/shots");
assertEquals(logRoot, jobLogUtil.getJobLogRootDir());
}

@Test
public void testGetJobLogDir() {
assertEquals(JobLogUtil.getJobLogDir("show", "shot"), "/shots/show/shot/logs");
assertEquals(logRoot + "/show/shot/logs", jobLogUtil.getJobLogDir("show", "shot"));
}

@Test
public void testGetJobLogPath() {
JobDetail jobDetail = new JobDetail();
jobDetail.id = "id";
jobDetail.name = "name";
jobDetail.showName = "show";
jobDetail.shot = "shot";
assertEquals(JobLogUtil.getJobLogPath(jobDetail), "/shots/show/shot/logs/name--id");
assertEquals(logRoot + "/show/shot/logs/name--id", jobLogUtil.getJobLogPath(jobDetail));
}
}

2 changes: 2 additions & 0 deletions cuebot/src/test/resources/opencue.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ grpc.rqd_cache_size=500
# RQD Channel Cache Expiration in Minutes
grpc.rqd_cache_expiration=30

log.frame-log-root=/arbitraryLogDirectory

dispatcher.job_query_max=20
dispatcher.job_lock_expire_seconds=2
dispatcher.job_lock_concurrency_level=3
Expand Down

0 comments on commit 4b7b789

Please sign in to comment.