message: compatible with redis protocol

diff:
*/
enum Sort
{
+ Unsort, //default
Asc,
Desc
};
@@ -117,6 +118,7 @@ private:
range_.count = false;
range_.option_num = 0;
range_.count_limit = 0;
+ range_.sort = Unsort;
}
};

diff:
@@ -392,13 +392,13 @@ void InitCmdInfoTable() {
CmdInfo* geoaddptr = new CmdInfo(kCmdNameGeoAdd, -4, kCmdFlagsWrite | kCmdFlagsGeo);
cmd_infos.insert(std::pair<std::string, CmdInfo*>(kCmdNameGeoAdd, geoaddptr));
////GeoPos
- CmdInfo* geoposptr = new CmdInfo(kCmdNameGeoPos, -3, kCmdFlagsRead | kCmdFlagsGeo);
+ CmdInfo* geoposptr = new CmdInfo(kCmdNameGeoPos, -2, kCmdFlagsRead | kCmdFlagsGeo);
cmd_infos.insert(std::pair<std::string, CmdInfo*>(kCmdNameGeoPos, geoposptr));
////GeoDist
CmdInfo* geodistptr = new CmdInfo(kCmdNameGeoDist, -3, kCmdFlagsRead | kCmdFlagsGeo);
cmd_infos.insert(std::pair<std::string, CmdInfo*>(kCmdNameGeoDist, geodistptr));
////GeoHash
- CmdInfo* geohashptr = new CmdInfo(kCmdNameGeoHash, -3, kCmdFlagsRead | kCmdFlagsGeo);
+ CmdInfo* geohashptr = new CmdInfo(kCmdNameGeoHash, -2, kCmdFlagsRead | kCmdFlagsGeo);
cmd_infos.insert(std::pair<std::string, CmdInfo*>(kCmdNameGeoHash, geohashptr));
////GeoRadius
CmdInfo* georadiusptr = new CmdInfo(kCmdNameGeoRadius, -3, kCmdFlagsRead | kCmdFlagsGeo);

diff:
@@ -21,7 +21,7 @@ void GeoAddCmd::DoInitial(PikaCmdArgsType &argv, const CmdInfo* const ptr_info)
}
size_t argc = argv.size();
if ((argc - 2) % 3 != 0) {
- res_.SetRes(CmdRes::kSyntaxErr);
+ res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoAdd);
return;
}
key_ = argv[1];
@@ -126,7 +126,7 @@ void GeoPosCmd::Do() {
}
}
-static double length_converter(double meters, std::string unit) {
+static double length_converter(double meters, const std::string & unit) {
if (unit == "m") {
return meters;
} else if (unit == "km") {
@@ -140,21 +140,38 @@ static double length_converter(double meters, std::string unit) {
}
}
+static bool check_unit(const std::string & unit) {
+ if (unit == "m" || unit == "km" || unit == "ft" || unit == "mi") {
+ return true;
+ } else {
+ return false;
+ }
+}
+
void GeoDistCmd::DoInitial(PikaCmdArgsType &argv, const CmdInfo* const ptr_info) {
if (!ptr_info->CheckArg(argv.size())) {
res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoDist);
return;
}
+ if (argv.size() < 4) {
+ res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoDist);
+ return;
+ } else if (argv.size() > 5) {
+ res_.SetRes(CmdRes::kSyntaxErr);
+ return;
+ }
key_ = argv[1];
first_pos_ = argv[2];
second_pos_ = argv[3];
if (argv.size() == 5) {
unit_ = argv[4];
- } else if (argv.size() > 5) {
- res_.SetRes(CmdRes::kSyntaxErr);
} else {
unit_ = "m";
}
+ if (!check_unit(unit_)) {
+ res_.SetRes(CmdRes::kErrOther, "unsupported unit provided. please use m, km, ft, mi");
+ return;
+ }
}
void GeoDistCmd::Do() {
@@ -185,10 +202,6 @@ void GeoDistCmd::Do() {
double distance = geohashGetDistance(first_xy[0], first_xy[1], second_xy[0], second_xy[1]);
distance = length_converter(distance, unit_);
- if (distance == -1) {
- res_.SetRes(CmdRes::kErrOther, "unsupported unit provided. please use m, km, ft, mi");
- return;
- }
char buf[32];
int64_t len = slash::d2string(buf, sizeof(buf), distance);
res_.AppendStringLen(len);
@@ -258,7 +271,7 @@ static void GetAllNeighbors(std::string & key, GeoRange & range, CmdRes & res) {
nemo::Status s;
double longitude = range.longitude, latitude = range.latitude, distance = range.distance;
int count_limit = 0;
-
+ // Convert other units to meters
if (range.unit == "m") {
distance = distance;
} else if (range.unit == "km") {
@@ -268,8 +281,7 @@ static void GetAllNeighbors(std::string & key, GeoRange & range, CmdRes & res) {
} else if (range.unit == "mi") {
distance = distance * 1609.34;
} else {
- res.SetRes(CmdRes::kErrOther, "unsupported unit provided. please use m, km, ft, mi");
- return;
+ distance = -1;
}
// Search the zset for all matching points
GeoHashRadius georadius = geohashGetAreasByRadiusWGS84(longitude, latitude, distance);
@@ -344,10 +356,6 @@ static void GetAllNeighbors(std::string & key, GeoRange & range, CmdRes & res) {
geohashDecodeToLongLatWGS84(hash, xy);
double distance = geohashGetDistance(longitude, latitude, xy[0], xy[1]);
distance = length_converter(distance, range.unit);
- if (distance == -1) {
- res.SetRes(CmdRes::kErrOther, "unsupported unit provided. please use m, km, ft, mi");
- return;
- }
char buf[32];
int64_t len = slash::d2string(buf, sizeof(buf), distance);
res.AppendStringLen(len);
@@ -395,6 +403,10 @@ void GeoRadiusCmd::DoInitial(PikaCmdArgsType &argv, const CmdInfo* const ptr_inf
slash::string2d(argv[3].data(), argv[3].size(), &range_.latitude);
slash::string2d(argv[4].data(), argv[4].size(), &range_.distance);
range_.unit = argv[5];
+ if (!check_unit(range_.unit)) {
+ res_.SetRes(CmdRes::kErrOther, "unsupported unit provided. please use m, km, ft, mi");
+ return;
+ }
size_t pos = 6;
while (pos < argv.size()) {
if (!strcasecmp(argv[pos].c_str(), "withdist")) {
@@ -439,6 +451,10 @@ void GeoRadiusByMemberCmd::DoInitial(PikaCmdArgsType &argv, const CmdInfo* const
range_.member = argv[2];
slash::string2d(argv[3].data(), argv[3].size(), &range_.distance);
range_.unit = argv[4];
+ if (!check_unit(range_.unit)) {
+ res_.SetRes(CmdRes::kErrOther, "unsupported unit provided. please use m, km, ft, mi");
+ return;
+ }
size_t pos = 5;
while (pos < argv.size()) {
if (!strcasecmp(argv[pos].c_str(), "withdist")) {

original_message: compatible with redis protocol


message: Add 360game logo

change_type: MODIFY

diff:
@@ -12,23 +12,24 @@ Pika is a persistent huge storage service , compatible with the vast majority o

+

-

+

-

+

-

+

license: BSD 3-Clause New or Revised License

original_message: Add 360game logo


message: rsync optimized for speed

diff:
@@ -686,24 +686,25 @@ void PikaServer::DBSyncSendFile(const std::string& ip, int port) {
}
// Get all files need to send
std::vector<std::string> descendant;
- if (!slash::GetDescendant(bg_path, descendant)) {
- LOG(WARNING) << "Get Descendant when try to do db sync failed";
+ if (!slash::GetChildren(bg_path, descendant)) {
+ LOG(WARNING) << "Get child directory when try to do db sync failed";
}
// Iterate to send files
int ret = 0;
- std::string target_path;
+ std::string local_path, target_path;
std::string module = kDBSyncModule + "_" + slash::IpPortString(host_, port_);
std::vector<std::string>::iterator it = descendant.begin();
slash::RsyncRemote remote(ip, port, module, g_pika_conf->db_sync_speed() * 1024);
for (; it != descendant.end(); ++it) {
- target_path = (*it).substr(bg_path.size() + 1);
+ local_path = bg_path + "/" + *it;
+ target_path = *it;
+
if (target_path == kBgsaveInfoFile) {
continue;
}
// We need specify the speed limit for every single file
- ret = slash::RsyncSendFile(*it, target_path, remote);
-
+ ret = slash::RsyncSendFile(local_path, target_path, remote);
if (0 != ret) {
LOG(WARNING) << "rsync send file failed! From: " << *it

original_message: rsync optimized for speed (#162)


diff:
@@ -130,6 +130,15 @@ class PikaServer {
void Exit() {
exit_ = true;
}
+
+ void SetBinlogIoError(bool error) {
+ binlog_io_error_ = error;
+ }
+
+ bool BinlogIoError() {
+ return binlog_io_error_;
+ }
+
void DoTimingTask();
void Cleanup();
@@ -334,6 +343,7 @@ class PikaServer {
private:
std::atomic<bool> exit_;
+ std::atomic<bool> binlog_io_error_;
std::string host_;
int port_;
pthread_rwlock_t rwlock_;

diff:
@@ -296,7 +296,10 @@ Status Binlog::Produce(const Slice &item, int *temp_pro_offset) {
assert(leftover >= 0);
if (static_cast<size_t>(leftover) < kHeaderSize) {
if (leftover > 0) {
- queue_->Append(Slice("\x00\x00\x00\x00\x00\x00\x00", leftover));
+ s = queue_->Append(Slice("\x00\x00\x00\x00\x00\x00\x00", leftover));
+ if (!s.ok()) {
+ return s;
+ }
//version_->rise_pro_offset(leftover);
*temp_pro_offset += leftover;
//version_->StableSave();

diff:
@@ -92,6 +92,9 @@ std::string PikaClientConn::DoCmd(const std::string& opt) {
std::string raw_args;
if (cinfo_ptr->is_write()) {
+ if (g_pika_server->BinlogIoError()) {
+ return "-ERR Writing binlog failed, maybe no space left on device\r\n";
+ }
if (g_pika_conf->readonly()) {
return "-ERR Server in read-only\r\n";
}
@@ -111,8 +114,14 @@ std::string PikaClientConn::DoCmd(const std::string& opt) {
if (cinfo_ptr->is_write()) {
if (c_ptr->res().ok()) {
g_pika_server->logger_->Lock();
- g_pika_server->logger_->Put(raw_args);
+ slash::Status s = g_pika_server->logger_->Put(raw_args);
g_pika_server->logger_->Unlock();
+ if (!s.ok()) {
+ LOG(INFO) << "Writing binlog failed, maybe no space left on device";
+ g_pika_server->SetBinlogIoError(true);
+ g_pika_conf->SetReadonly(true);
+ return "-ERR Writing binlog failed, maybe no space left on device\r\n";
+ }
}
}

diff:
@@ -30,6 +30,7 @@ extern PikaConf *g_pika_conf;
PikaServer::PikaServer() :
ping_thread_(NULL),
exit_(false),
+ binlog_io_error_(false),
have_scheduled_crontask_(false),
last_check_compact_time_({0, 0}),
sid_(0),

original_message: fix when the disk is full cause pika crash (#190)


message: add CPU usage

diff:
#define PIKA_ADMIN_H_
#include <sstream>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <iomanip>
#include "include/pika_command.h"
#include "include/pika_client_conn.h"
@@ -169,6 +172,7 @@ class InfoCmd : public Cmd {
kInfoClients,
kInfoHub,
kInfoStats,
+ kInfoCPU,
kInfoReplication,
kInfoKeyspace,
kInfoBgstats,
@@ -191,6 +195,7 @@ class InfoCmd : public Cmd {
const static std::string kClientsSection;
const static std::string kHubSection;
const static std::string kStatsSection;
+ const static std::string kCPUSection;
const static std::string kReplicationSection;
const static std::string kKeyspaceSection;
const static std::string kLogSection;
@@ -207,6 +212,7 @@ class InfoCmd : public Cmd {
void InfoClients(std::string &info);
void InfoHub(std::string &info);
void InfoStats(std::string &info);
+ void InfoCPU(std::string &info);
void InfoReplication(std::string &info);
void InfoKeyspace(std::string &info);
void InfoLog(std::string &info);

diff:
@@ -473,6 +473,7 @@ const std::string InfoCmd::kServerSection = "server";
const std::string InfoCmd::kClientsSection = "clients";
const std::string InfoCmd::kHubSection = "hub";
const std::string InfoCmd::kStatsSection = "stats";
+const std::string InfoCmd::kCPUSection = "cpu";
const std::string InfoCmd::kReplicationSection = "replication";
const std::string InfoCmd::kKeyspaceSection = "keyspace";
const std::string InfoCmd::kLogSection = "log";
@@ -501,6 +502,8 @@ void InfoCmd::DoInitial(PikaCmdArgsType &argv, const CmdInfo* const ptr_info) {
info_section_ = kInfoHub;
} else if (argv[1] == kStatsSection) {
info_section_ = kInfoStats;
+ } else if (argv[1] == kCPUSection) {
+ info_section_ = kInfoCPU;
} else if (argv[1] == kReplicationSection) {
info_section_ = kInfoReplication;
} else if (argv[1] == kKeyspaceSection) {
@@ -546,6 +549,8 @@ void InfoCmd::Do() {
info.append("\r\n");
InfoStats(info);
info.append("\r\n");
+ InfoCPU(info);
+ info.append("\r\n");
InfoReplication(info);
info.append("\r\n");
InfoKeyspace(info);
@@ -564,6 +569,9 @@ void InfoCmd::Do() {
case kInfoStats:
InfoStats(info);
break;
+ case kInfoCPU:
+ InfoCPU(info);
+ break;
case kInfoReplication:
InfoReplication(info);
break;
@@ -672,6 +680,31 @@ void InfoCmd::InfoStats(std::string &info) {
info.append(tmp_stream.str());
}
+void InfoCmd::InfoCPU(std::string &info) {
+ struct rusage self_ru, c_ru;
+ getrusage(RUSAGE_SELF, &self_ru);
+ getrusage(RUSAGE_CHILDREN, &c_ru);
+ std::stringstream tmp_stream;
+ tmp_stream << "# CPU\r\n";
+ tmp_stream << "used_cpu_sys:" <<
+ setiosflags(std::ios::fixed) << std::setprecision(2) <<
+ (float)self_ru.ru_stime.tv_sec+(float)self_ru.ru_stime.tv_usec/1000000 <<
+ "\r\n";
+ tmp_stream << "used_cpu_user:" <<
+ setiosflags(std::ios::fixed) << std::setprecision(2) <<
+ (float)self_ru.ru_utime.tv_sec+(float)self_ru.ru_utime.tv_usec/1000000 <<
+ "\r\n";
+ tmp_stream << "used_cpu_sys_children:" <<
+ setiosflags(std::ios::fixed) << std::setprecision(2) <<
+ (float)c_ru.ru_stime.tv_sec+(float)c_ru.ru_stime.tv_usec/1000000 <<
+ "\r\n";
+ tmp_stream << "used_cpu_user_children:" <<
+ setiosflags(std::ios::fixed) << std::setprecision(2) <<
+ (float)c_ru.ru_utime.tv_sec+(float)c_ru.ru_utime.tv_usec/1000000 <<
+ "\r\n";
+ info.append(tmp_stream.str());
+}
+
void InfoCmd::InfoReplication(std::string &info) {
int host_role = g_pika_server->role();
std::stringstream tmp_stream;

original_message: add CPU usage (#218)


message: support multiple compression algorithm

diff:
@@ -67,8 +67,13 @@ PikaServer::PikaServer() :
option.max_open_files = g_pika_conf->max_cache_files();
option.max_bytes_for_level_multiplier = g_pika_conf->max_bytes_for_level_multiplier();
if (g_pika_conf->compression() == "none") {
- option.compression = false;
+ option.compression = nemo::Options::CompressionType::kNoCompression;
+ } else if (g_pika_conf->compression() == "snappy") {
+ option.compression = nemo::Options::CompressionType::kSnappyCompression;
+ } else if (g_pika_conf->compression() == "zlib") {
+ option.compression = nemo::Options::CompressionType::kZlibCompression;
}
+
std::string db_path = g_pika_conf->db_path();
LOG(INFO) << "Prepare DB...";
db_ = std::shared_ptr<nemo::Nemo>(new nemo::Nemo(db_path, option));
@@ -393,7 +398,11 @@ bool PikaServer::ChangeDb(const std::string& new_path) {
option.max_open_files = g_pika_conf->max_cache_files();
option.max_bytes_for_level_multiplier = g_pika_conf->max_bytes_for_level_multiplier();
if (g_pika_conf->compression() == "none") {
- option.compression = false;
+ option.compression = nemo::Options::CompressionType::kNoCompression;
+ } else if (g_pika_conf->compression() == "snappy") {
+ option.compression = nemo::Options::CompressionType::kSnappyCompression;
+ } else if (g_pika_conf->compression() == "zlib") {
+ option.compression = nemo::Options::CompressionType::kZlibCompression;
}
std::string db_path = g_pika_conf->db_path();
std::string tmp_path(db_path);
@@ -1395,8 +1404,13 @@ bool PikaServer::FlushAll() {
option.max_open_files = g_pika_conf->max_cache_files();
option.max_bytes_for_level_multiplier = g_pika_conf->max_bytes_for_level_multiplier();
if (g_pika_conf->compression() == "none") {
- option.compression = false;
+ option.compression = nemo::Options::CompressionType::kNoCompression;
+ } else if (g_pika_conf->compression() == "snappy") {
+ option.compression = nemo::Options::CompressionType::kSnappyCompression;
+ } else if (g_pika_conf->compression() == "zlib") {
+ option.compression = nemo::Options::CompressionType::kZlibCompression;
}
+
LOG(INFO) << "Prepare open new db...";
db_ = std::shared_ptr<nemo::Nemo>(new nemo::Nemo(g_pika_conf->db_path(), option));
LOG(INFO) << "open new db success";

original_message: support multiple compression algorithm


message: bugfix master  slave auth failed

diff:
@@ -76,7 +76,7 @@ bool PikaTrysyncThread::Send() {
}
bool PikaTrysyncThread::RecvProc() {
- bool should_auth = g_pika_conf->requirepass() == "" ? false : true;
+ bool should_auth = g_pika_conf->masterauth() == "" ? false : true;
bool is_authed = false;
slash::Status s;
std::string reply;

original_message: bugfix master  slave auth failed


message: bugfix configurable make slave-priority configurable

diff:
@@ -1243,7 +1243,7 @@ void ConfigCmd::ConfigGet(std::string &ret) {
EncodeString(&ret, g_pika_conf->network_interface());
EncodeString(&ret, "slaveof");
EncodeString(&ret, g_pika_conf->slaveof());
- EncodeString(&ret, "slaveof-priority");
+ EncodeString(&ret, "slave-priority");
EncodeInt32(&ret, g_pika_conf->slave_priority());
} else {
ret = "*0\r\n";


message: bugfix Info format

diff:
@@ -818,7 +818,7 @@ void InfoCmd::InfoDoubleMaster(std::string &info) {
switch (host_role) {
case PIKA_ROLE_SINGLE :
case PIKA_ROLE_MASTER : tmp_stream << "MASTER)\r\nrole:master\r\n"; break;
- case PIKA_ROLE_SLAVE : tmp_stream << "SLAVE)\r\nrole:slave\r\b"; break;
+ case PIKA_ROLE_SLAVE : tmp_stream << "SLAVE)\r\nrole:slave\r\n"; break;
case PIKA_ROLE_DOUBLE_MASTER :
if (g_pika_server->DoubleMasterMode()) {
tmp_stream << "DOUBLEMASTER)\r\nrole:double_master\r\n"; break;

original_message: bugfix Info format


message: upgrade pink

old_path: .gitmodules

new_path: .gitmodules

diff:
url = https://github.com/Qihoo360/glog.git
[submodule "third/pink"]
path = third/pink
- url = https://github.com/Qihoo360/pink.git
+ url = https://github.com/PikaLabs/pink.git
[submodule "third/slash"]
path = third/slash
url = https://github.com/Qihoo360/slash.git

old_path: third/pink

new_path: third/pink

diff:
-Subproject commit 51e4bbeed2c9dbf2f728400369fc9d19d482bfe2
+Subproject commit 27a24b76fed350e42611ab04bac3ef5328f9470a

original_message: upgrade pink


message:
bugfix binlog sender confict at multi-network interface
* bugfix binlog sender confict at multi-network interface
* bugfix configing set maxclients has no effect
* set resource
limits

diff:
@@ -20,6 +20,7 @@ class PikaBinlogSenderThread : public pink::Thread {
public:
PikaBinlogSenderThread(const std::string &ip, int port,
+ int64_t sid,
slash::SequentialFile *queue,
uint32_t filenum, uint64_t con_offset);
@@ -50,6 +51,7 @@ class PikaBinlogSenderThread : public pink::Thread {
std::string ip_;
int port_;
+ int64_t sid_;
int timeout_ms_;
pink::PinkCli *cli_;

diff:
@@ -22,6 +22,10 @@ class PikaDispatchThread {
bool ClientKill(const std::string& ip_port);
void ClientKillAll();
+ void SetQueueLimit(int queue_limit) {
+ thread_rep_->SetQueueLimit(queue_limit);
+ }
+
private:
class ClientConnFactory : public pink::ConnFactory {
public:

diff:
@@ -16,6 +16,7 @@ class PikaMasterConn: public pink::RedisConn {
PikaMasterConn(int fd, std::string ip_port, void* worker_specific_data);
virtual int DealMessage(PikaCmdArgsType& argv, std::string* response);
private:
+ bool is_first_send_;
PikaBinlogReceiverThread* binlog_receiver_;
};

diff:
@@ -70,6 +70,15 @@ class PikaServer {
int master_port() {
return master_port_;
}
+
+ int64_t sid() {
+ return sid_;
+ }
+
+ void SetSid(int64_t sid) {
+ sid_ = sid;
+ }
+
const std::shared_ptr<nemo::Nemo> db() {
return db_;
}
@@ -184,6 +193,7 @@ class PikaServer {
*/
Binlog *logger_;
Status AddBinlogSender(const std::string& ip, int64_t port,
+ int64_t sid,
uint32_t filenum, uint64_t con_offset);
/*
@@ -457,6 +467,8 @@ class PikaServer {
void ResetStat();
slash::RecordMutex mutex_record_;
+ void SetDispatchQueueLimit(int queue_limit);
+
private:
std::atomic<bool> exit_;
std::atomic<bool> binlog_io_error_;

diff:
@@ -163,6 +163,7 @@ void TrysyncCmd::Do() {
int64_t sid = g_pika_server->TryAddSlave(slave_ip_, slave_port_);
if (sid >= 0) {
Status status = g_pika_server->AddBinlogSender(slave_ip_, slave_port_,
+ sid,
filenum_, pro_offset_);
if (status.ok()) {
res_.AppendInteger(sid);
@@ -1320,6 +1321,7 @@ void ConfigCmd::ConfigSet(std::string& ret) {
ret = "-ERR Invalid argument " + value + " for CONFIG SET 'maxclients'\r\n";
return;
}
+ g_pika_server->SetDispatchQueueLimit(ival);
g_pika_conf->SetMaxConnection(ival);
ret = "+OK\r\n";
} else if (set_item == "dump-expire") {

diff:
@@ -39,10 +39,10 @@ bool PikaBinlogReceiverThread::Handles::AccessHandle(std::string& ip) const {
ip = g_pika_server->host();
}
// if (binlog_receiver_->thread_rep_->conn_num() != 0 ||
- if (!g_pika_server->ShouldAccessConnAsMaster(ip)) {
- LOG(WARNING) << "BinlogReceiverThread AccessHandle failed: " << ip;
- return false;
- }
+ // if (!g_pika_server->ShouldAccessConnAsMaster(ip)) {
+ // LOG(WARNING) << "BinlogReceiverThread AccessHandle failed: " << ip;
+ // return false;
+ // }
g_pika_server->PlusMasterConnection();
return true;
}

diff:
extern PikaServer* g_pika_server;
PikaBinlogSenderThread::PikaBinlogSenderThread(const std::string &ip, int port,
+ int64_t sid,
slash::SequentialFile *queue,
uint32_t filenum,
uint64_t con_offset)
@@ -29,6 +30,7 @@ PikaBinlogSenderThread::PikaBinlogSenderThread(const std::string &ip, int port,
buffer_(),
ip_(ip),
port_(port),
+ sid_(sid),
timeout_ms_(35000) {
cli_ = pink::NewRedisCli();
last_record_offset_ = con_offset % kBlockSize;
@@ -257,6 +259,17 @@ void* PikaBinlogSenderThread::ThreadMain() {
if (result.ok()) {
cli_->set_send_timeout(timeout_ms_);
+ // Auth sid
+ std::string wbuf_str;
+ pink::RedisCmdArgsType argv;
+ argv.push_back("auth");
+ argv.push_back(std::to_string(sid_));
+ pink::SerializeRedisCommand(argv, &wbuf_str);
+ result = cli_->Send(&wbuf_str);
+ if (!result.ok()) {
+ LOG(WARNING) << "BinlogSender send slave(" << ip_ << ":" << port_ << ") failed, " << result.ToString();
+ break;
+ }
while (true) {
// 2. Should Parse new msg;
if (last_send_flag) {

diff:
@@ -17,6 +17,7 @@ extern PikaConf* g_pika_conf;
PikaMasterConn::PikaMasterConn(int fd, std::string ip_port,
void* worker_specific_data)
: RedisConn(fd, ip_port, NULL) {
+ is_first_send_ = true;
binlog_receiver_ =
reinterpret_cast<PikaBinlogReceiverThread*>(worker_specific_data);
}
@@ -30,6 +31,19 @@ int PikaMasterConn::DealMessage(
return -2;
}
+ // Auth
+ if (is_first_send_) {
+ if (argv[0] == "auth") {
+ if (argv[1] == std::to_string(g_pika_server->sid())) {
+ is_first_send_ = false;
+ LOG(INFO) << "BinlogReceiverThread AccessHandle success, my sid: " << g_pika_server->sid() << " auth sid: " << argv[1];
+ return 0;
+ }
+ }
+ LOG(WARNING) << "BinlogReceiverThread AccessHandle failed";
+ return -2;
+ }
+
// TODO(shq) maybe monitor do not need these infomation
std::string server_id;
std::string binlog_info;

diff:
#include <iterator>
#include <ctime>
#include <algorithm>
+#include <sys/resource.h>
#include "slash/include/env.h"
#include "slash/include/rsync.h"
@@ -850,6 +851,7 @@ void PikaServer::DBSyncSendFile(const std::string& ip, int port) {
* BinlogSender
*/
Status PikaServer::AddBinlogSender(const std::string& ip, int64_t port,
+ int64_t sid,
uint32_t filenum, uint64_t con_offset) {
// Sanity check
if (con_offset > logger_->file_size()) {
@@ -885,7 +887,7 @@ Status PikaServer::AddBinlogSender(const std::string& ip, int64_t port,
}
PikaBinlogSenderThread* sender = new PikaBinlogSenderThread(ip,
- port + 1000, readfile, filenum, con_offset);
+ port + 1000, sid, readfile, filenum, con_offset);
if (sender->trim() == 0 // Error binlog
&& SetSlaveSender(ip, port, sender)) { // SlaveItem not exist
@@ -1720,6 +1722,25 @@ void PikaServer::ResetStat() {
statistic_data_.last_thread_querynum = 0;
}
+void PikaServer::SetDispatchQueueLimit(int queue_limit) {
+ rlimit limit;
+ if (getrlimit(RLIMIT_NOFILE,&limit) == -1) {
+ LOG(WARNING) << "getrlimit error: " << strerror(errno);
+ } else if (limit.rlim_cur < static_cast<unsigned int>(g_pika_conf->maxclients() + PIKA_MIN_RESERVED_FDS)) {
+ rlim_t old_limit = limit.rlim_cur;
+ rlim_t best_limit = g_pika_conf->maxclients() + PIKA_MIN_RESERVED_FDS;
+ limit.rlim_cur = best_limit > limit.rlim_max ? limit.rlim_max-1 : best_limit;
+ limit.rlim_max = best_limit > limit.rlim_max ? limit.rlim_max-1 : best_limit;
+ if (setrlimit(RLIMIT_NOFILE,&limit) != -1) {
+ LOG(WARNING) << "your 'limit -n ' of " << old_limit << " is not enough for Redis to start. pika have successfully reconfig it to " << limit.rlim_cur;
+ } else {
+ LOG(FATAL) << "your 'limit -n ' of " << old_limit << " is not enough for Redis to start. pika can not reconfig it(" << strerror(errno) << "), do it by yourself";
+ }
+ }
+
+ pika_dispatch_thread_->SetQueueLimit(queue_limit);
+}
+
uint64_t PikaServer::ServerCurrentQps() {
slash::ReadLock l(&statistic_data_.statistic_lock);
return statistic_data_.last_sec_thread_querynum;

diff:
@@ -104,6 +104,7 @@ bool PikaTrysyncThread::RecvProc() {
slash::string2l(reply.data(), reply.size(), &sid_)) {
// Luckly, I got your point, the sync is comming
LOG(INFO) << "Recv sid from master: " << sid_;
+ g_pika_server->SetSid(sid_);
break;
}

original_message:
bugfix binlog sender confict at multi-network interface (#234)

* bugfix binlog sender confict at multi-network interface

* bugfix configing set maxclients has no effect

* set resource
       limits


message: optimize confict at multi-network interface

diff:
@@ -257,6 +257,7 @@ void* PikaBinlogSenderThread::ThreadMain() {
result = cli_->Connect(ip_, port_, g_pika_server->host());
LOG(INFO) << "BinlogSender Connect slave(" << ip_ << ":" << port_ << ") " << result.ToString();
+ // 2. Auth
if (result.ok()) {
cli_->set_send_timeout(timeout_ms_);
// Auth sid
@@ -271,7 +272,7 @@ void* PikaBinlogSenderThread::ThreadMain() {
break;
}
while (true) {
- // 2. Should Parse new msg;
+ // 3. Should Parse new msg;
if (last_send_flag) {
s = Parse(scratch);
//DLOG(INFO) << "BinlogSender Parse, return " << s.ToString();
@@ -303,7 +304,7 @@ void* PikaBinlogSenderThread::ThreadMain() {
continue;
}
- // 3. After successful parse, we send msg;
+ // 4. After successful parse, we send msg;
result = cli_->Send(&scratch);
if (result.ok()) {
last_send_flag = true;

diff:
@@ -36,7 +36,7 @@ int PikaMasterConn::DealMessage(
if (argv[0] == "auth") {
if (argv[1] == std::to_string(g_pika_server->sid())) {
is_first_send_ = false;
- LOG(INFO) << "BinlogReceiverThread AccessHandle success, my sid: " << g_pika_server->sid() << " auth sid: " << argv[1];
+ LOG(INFO) << "BinlogReceiverThread AccessHandle succeeded, My server id: " << g_pika_server->sid() << " Master auth server id: " << argv[1];
return 0;
}
}


message: Support disconnect for double master

diff:
@@ -89,7 +89,7 @@ void SlaveofCmd::Do() {
}
// Stop rsync
- LOG(INFO) << "start slaveof, stop rsync first";
+ LOG(INFO) << "Start slaveof, stop rsync first";
slash::StopRsync(g_pika_conf->db_sync_path());
g_pika_server->RemoveMaster();
@@ -105,6 +105,22 @@ void SlaveofCmd::Do() {
}
g_pika_server->logger_->SetProducerStatus(filenum_, pro_offset_);
}
+ } else {
+ if (is_noone_) {
+ // Stop rsync
+ LOG(INFO) << "Slaveof no one in double-master mode";
+ slash::StopRsync(g_pika_conf->db_sync_path());
+
+ g_pika_server->RemoveMaster();
+
+ std::string double_master_ip = g_pika_conf->double_master_ip();
+ if (double_master_ip == "127.0.0.1") {
+ double_master_ip = g_pika_server->host();
+ }
+ g_pika_server->DeleteSlave(double_master_ip, g_pika_conf->double_master_port());
+ res_.SetRes(CmdRes::kOk);
+ return;
+ }
}
// The conf file already configured double-master item, but now this

diff:
@@ -324,7 +324,7 @@ void PikaServer::Start() {
int32_t double_master_port = g_pika_conf->double_master_port();
double_master_sid_ = std::stoi(g_pika_conf->double_master_sid());
if ((double_master_ip == "127.0.0.1" || double_master_ip == host_) && double_master_port == port_) {
- LOG(FATAL) << "set yourself as the peer-master, please check";
+ LOG(FATAL) << "Set yourself as the peer-master, please check";
} else {
double_master_mode_ = true;
SetMaster(double_master_ip, double_master_port);
@@ -355,16 +355,9 @@ void PikaServer::DeleteSlave(const std::string& ip, int64_t port) {
}
iter++;
}
- if (iter == slaves_.end()) {
- return;
- }
- if (iter->sender != NULL) {
- delete static_cast<PikaBinlogSenderThread*>(iter->sender);
- }
- slaves_.erase(iter);
- slave_num = slaves_.size();
- }
+ slave_num = slaves_.size();
+ {
slash::RWLock l(&state_protector_, true);
if (slave_num == 0) {
role_ &= ~PIKA_ROLE_MASTER;
@@ -374,6 +367,16 @@ void PikaServer::DeleteSlave(const std::string& ip, int64_t port) {
}
}
+ if (iter == slaves_.end()) {
+ return;
+ }
+ if (iter->sender != NULL) {
+ delete static_cast<PikaBinlogSenderThread*>(iter->sender);
+ }
+ slaves_.erase(iter);
+ }
+}
+
void PikaServer::DeleteSlave(int fd) {
int slave_num = 0;
{
@@ -706,7 +709,6 @@ void PikaServer::SyncError() {
}
void PikaServer::RemoveMaster() {
-
{
slash::RWLock l(&state_protector_, true);
repl_state_ = PIKA_REPL_NO_CONNECT;
@@ -722,7 +724,7 @@ void PikaServer::RemoveMaster() {
if (ping_thread_ != NULL) {
int err = ping_thread_->StopThread();
if (err != 0) {
- std::string msg = "can't join thread " + std::string(strerror(err));
+ std::string msg = "Can't join thread " + std::string(strerror(err));
LOG(WARNING) << msg;
}
delete ping_thread_;
@@ -732,7 +734,7 @@ void PikaServer::RemoveMaster() {
slash::RWLock l(&state_protector_, true);
master_connection_ = 0;
}
- LOG(INFO) << "close read-only mode";
+ LOG(INFO) << "Close read-only mode";
g_pika_conf->SetReadonly(false);
}

original_message: Support disconnect for double master (#381)


message:
added OS-sensitive vars for httpd
Debian requires apache2, RHEL variants req httpd
added dict variables for {apache2,httpd}.conf location
and service name, with ansible_os_family {Debian,RedHat} as the keys.

diff:
@@ -4,7 +4,12 @@ pga_group: "pga"
doc_root_dir: "/var/www/html/php-gateway"
user_data_dir: "/var/www/user_data"
#httpd_conf_file: "/etc/httpd/conf/httpd.conf"
-httpd_conf_file_location: "/etc/httpd/conf"
+httpd_conf_file_location:
+ RedHat: "/etc/httpd/conf/httpd.conf"
+ Debian: "/etc/apache2/apache2.conf"
+httpd_name:
+ Debian: apache2
+ RedHat: httpd
## WSO2 IS related variables
tenant_domain: "prod.testdrive"

license: Apache License 2.0

repo: apache/airavata


message:
created Debian-specifig dependency include
include file for Debian-specific dependency installation
since package names are different; could also potentially
handle this via a list variable, but for now just getting
this to work.

change_type: ADD

diff:
+---
+
+- name: Install prerequisites
+ apt: name="{{ item }}" state=latest update_cache=yes
+ with_items:
+ - git
+ - apache2
+ - php5
+ - php-soap
+ - python-selinux
+ - openssl
+ - curl
+ - php5-mysql
+ - php5-mcrypt
+ become: yes
+
+- name: install composer
+ shell: curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer
+ args:
+ creates: /usr/local/bin/composer
+ become: yes
+
+# this just creates symlinks in /etc/php5/{apache2,cli}/conf.d/20-mcryp.ini to /etc/php5/modules-avail/mcrypt.ini
+- name: enable mcrypt
+ command: php5enmod mcrypt
+ become: yes
+
+- name: create apache2 logdir
+ file:
+ path: /etc/apache2/logs
+ state: directory
+ owner: www-data
+ group: www-data
+ mode: 0775
+


message:
RedHat specific depenedency installation include
contains tasks specific to RedHat installation of PGA,
notably calling yum vs apt, and specific package names.

diff:
+---
+
+- name: Install pre-requireties
+ yum: name="{{ item }}" state=latest update_cache=yes
+ with_items:
+ - git
+ - httpd
+ - php
+ - php-soap
+ - libselinux-python
+ - php-pdo
+ become: yes
+#"
+
+# - name: Allow selinux outbound connection from web server
+ # command: setsebool -P httpd_can_network_connect 1
+
+- name: install composer
+ shell: curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer
+ args:
+ creates: /usr/local/bin/composer
+ become: yes
+
+- name: install epel release
+ yum: name=epel-release state=present
+ become: yes
+
+#OS_dep
+# depend on epel release
+- name: install mcrypt
+ yum: name=php-mcrypt state=latest update_cache=yes
+ become: yes
+


message:
added switch to firewalld install for Debian/Rhel
switch between apt/yum for Debian vs Rhel; this could
be handled a variety of ways, but thought to keep this
file straightforward for now. Also added comment on iptables,
for which firewalld is just a frontend.

diff:
user: name={{ user }} group={{ group }}
become: yes
-- name: Install Firewalld
+- name: Install Firewalld (RedHat)
yum: name=firewalld state=latest update_cache=yes
become: yes
+ when: ansible_os_family == "RedHat"
+
+- name: Install Firewalld (Debian)
+ apt: name=firewalld state=latest update_cache=yes
+ become: yes
+ when: ansible_os_family == "Debian"
+
# TODO: stop iptables service, can't have both iptables and firewalld on same host
+# firewalld is just a frontend for iptables - so we can't remove it
# if we try to stop non existing service ansible fails.
# - name: Stop iptables, ip6tables services
# service: name="{{ item }}" state=stopped


message:
changed httpd name var to rely on OS
now using a dictionary with OS family as key to get the
correct name of the http daemon for Debian/RedHat.

diff:
become: yes
- name: start httpd
- service: name=httpd state=started
+ service: name={{ httpd_name[ansible_os_family] }} state=started
become: yes
- name: restart httpd
- service: name=httpd state=restarted
+ service: name={{ httpd_name[ansible_os_family] }} state=restarted
become: yes


message:
moved OS-specific commands to separate includes
moved dependency installs, firewalld into specific includes
dependent on ansible_os_family - so far these exist for
Debian and RedHat

diff:
file: path="{{ doc_root_dir }}" state=directory
become: yes
-- name: Install pre-requireties
- yum: name="{{ item }}" state=latest update_cache=yes
- with_items:
- - git
- - httpd
- - php
- - php-soap
- - libselinux-python
- - php-pdo
- become: yes
-
-# - name: Allow selinux outbound connection from web server
- # command: setsebool -P httpd_can_network_connect 1
-
-- name: install composer
- shell: curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer
- args:
- creates: /usr/local/bin/composer
- become: yes
-
-- name: install epel release
- yum: name=epel-release state=present
- become: yes
-
-# depend on epel release
-- name: install mcrypt
- yum: name=php-mcrypt state=latest update_cache=yes
- become: yes
+- include: install_deps_{{ ansible_os_family }}.yml
# - name: Check previous deployments
# stat: path="{{ doc_root_dir }}" get_md5=no get_checksum=no


message: removed extra line, spell-corrected enable

diff:
- name: delete previous deployments
file: path="{{ doc_root_dir }}" state=absent
-
- name: Git clone php gateway
git: repo=https://github.com/apache/airavata-php-gateway.git
dest="{{ doc_root_dir }}" version="{{ git_branch }}"
file: path="{{ doc_root_dir }}/app/storage" state=directory mode=0777 owner="{{user}}" group="{{group}}" recurse=yes
become: yes
-- name: Eanble https and http service on public zone
+- name: Enable https and http service on public zone
firewalld: service="{{ item }}" permanent=true state=enabled zone=public immediate=True
with_items:
- http


message:
changed to os-dependent httpd conf
Changed task to set http daemon config to use {Debian,RedHat}
dependent dictionary for template name and destination. Also
removed extraneous http daemon restart, since this is handily
handled by the handler.

diff:
# become: yes
- name: copy httpd.conf file
- template: src=httpd.conf.j2 dest="{{ httpd_conf_file_location }}/httpd.conf"
+ template: src={{ httpd_name[ansible_os_family] }}.conf.j2 dest={{ httpd_conf_file_location[ansible_os_family] }}
become: yes
notify:
- restart httpd
-- name: start httpd service
- service: name=httpd state=started
- become: yes
+#- name: start httpd service
+# service: name=httpd state=started
+# become: yes


message:
added correct dependencies for Debian install
Had to check the right package names for Debian build,
added to apt task.

diff:
with_items:
- git
- apache2
- - php5
- - php-soap
- python-selinux
+ - unzip
- openssl
- curl
+ - php5
+ - php5-curl
+ - php-soap
- php5-mysql
- php5-mcrypt
+ - python-software-properties
become: yes
- name: install composer


message:
working apache2 config
Had to enable mcrypt via php2enmod, rewrite mod via a2enmod, which
while using the command task is less than ideal, at least uses
platform-preferred methods. Also fixed spacing in the logdir
creation task, and set DocumentRoot in the default sites-enabled,
instead of the main apache2 config.

diff:
command: php5enmod mcrypt
become: yes
+- name: enable apache rewrite
+ command: a2enmod rewrite
+ become: yes
+
- name: create apache2 logdir
file:
path: /etc/apache2/logs
group: www-data
mode: 0775
+- name: set DocumentRoot in default site-enabled
+ lineinfile:
+ dest: /etc/apache2/sites-enabled/000-default.conf
+ line: ' DocumentRoot "/var/www/html/php-gateway/public"'
+ regexp: '^\s+DocumentRoot'
+ state: present


message:
added working apache2.conf
Debian-specific httpd daemon cocnfig file.

diff:
+# This is the main Apache server configuration file. It contains the
+# configuration directives that give the server its instructions.
+# See http://httpd.apache.org/docs/2.4/ for detailed information about
+# the directives and /usr/share/doc/apache2/README.Debian about Debian specific
+# hints.
+#
+#
+# Summary of how the Apache 2 configuration works in Debian:
+# The Apache 2 web server configuration in Debian is quite different to
+# upstream's suggested way to configure the web server. This is because Debian's
+# default Apache2 installation attempts to make adding and removing modules,
+# virtual hosts, and extra configuration directives as flexible as possible, in
+# order to make automating the changes and administering the server as easy as
+# possible.
+
+# It is split into several files forming the configuration hierarchy outlined
+# below, all located in the /etc/apache2/ directory:
+#
+# /etc/apache2/
+# |-- apache2.conf
+# | `-- ports.conf
+# |-- mods-enabled
+# | |-- *.load
+# | `-- *.conf
+# |-- conf-enabled
+# | `-- *.conf
+# `-- sites-enabled
+# `-- *.conf
+#
+#
+# * apache2.conf is the main configuration file (this file). It puts the pieces
+# together by including all remaining configuration files when starting up the
+# web server.
+#
+# * ports.conf is always included from the main configuration file. It is
+# supposed to determine listening ports for incoming connections which can be
+# customized anytime.
+#
+# * Configuration files in the mods-enabled/, conf-enabled/ and sites-enabled/
+# directories contain particular configuration snippets which manage modules,
+# global configuration fragments, or virtual host configurations,
+# respectively.
+#
+# They are activated by symlinking available configuration files from their
+# respective *-available/ counterparts. These should be managed by using our
+# helpers a2enmod/a2dismod, a2ensite/a2dissite and a2enconf/a2disconf. See
+# their respective man pages for detailed information.
+#
+# * The binary is called apache2. Due to the use of environment variables, in
+# the default configuration, apache2 needs to be started/stopped with
+# /etc/init.d/apache2 or apache2ctl. Calling /usr/bin/apache2 directly will not
+# work with the default configuration.
+
+
+# Global configuration
+#
+
+#
+# ServerRoot: The top of the directory tree under which the server's
+# configuration, error, and log files are kept.
+#
+# NOTE! If you intend to place this on an NFS (or otherwise network)
+# mounted filesystem then please read the Mutex documentation (available
+# at <URL:http://httpd.apache.org/docs/2.4/mod/core.html#mutex>);
+# you will save yourself a lot of trouble.
+#
+# Do NOT add a slash at the end of the directory path.
+#
+#ServerRoot "/etc/apache2"
+DocumentRoot "/var/www/html/php-gateway/public"
+
+#
+# The accept serialization lock file MUST BE STORED ON A LOCAL DISK.
+#
+Mutex file:${APACHE_LOCK_DIR} default
+
+#
+# PidFile: The file in which the server should record its process
+# identification number when it starts.
+# This needs to be set in /etc/apache2/envvars
+#
+PidFile ${APACHE_PID_FILE}
+
+#
+# Timeout: The number of seconds before receives and sends time out.
+#
+Timeout 300
+
+#
+# KeepAlive: Whether or not to allow persistent connections (more than
+# one request per connection). Set to "Off" to deactivate.
+#
+KeepAlive On
+
+#
+# MaxKeepAliveRequests: The maximum number of requests to allow
+# during a persistent connection. Set to 0 to allow an unlimited amount.
+# We recommend you leave this number high, for maximum performance.
+#
+MaxKeepAliveRequests 100
+
+#
+# KeepAliveTimeout: Number of seconds to wait for the next request from the
+# same client on the same connection.
+#
+KeepAliveTimeout 5
+
+
+# These need to be set in /etc/apache2/envvars
+User ${APACHE_RUN_USER}
+Group ${APACHE_RUN_GROUP}
+
+#
+# HostnameLookups: Log the names of clients or just their IP addresses
+# e.g., www.apache.org (on) or 204.62.129.132 (off).
+# The default is off because it'd be overall better for the net if people
+# had to knowingly turn this feature on, since enabling it means that
+# each client request will result in AT LEAST one lookup request to the
+# nameserver.
+#
+HostnameLookups Off
+
+# ErrorLog: The location of the error log file.
+# If you do not specify an ErrorLog directive within a <VirtualHost>
+# container, error messages relating to that virtual host will be
+# logged here. If you *do* define an error logfile for a <VirtualHost>
+# container, that host's errors will be logged there and not here.
+#
+ErrorLog ${APACHE_LOG_DIR}/error.log
+
+#
+# LogLevel: Control the severity of messages logged to the error_log.
+# Available values: trace8, ..., trace1, debug, info, notice, warn,
+# error, crit, alert, emerg.
+# It is also possible to configure the log level for particular modules, e.g.
+# "LogLevel info ssl:warn"
+#
+LogLevel warn
+
+# Include module configuration:
+IncludeOptional mods-enabled/*.load
+IncludeOptional mods-enabled/*.conf
+
+# Include list of ports to listen on
+Include ports.conf
+
+
+# Sets the default security model of the Apache2 HTTPD server. It does
+# not allow access to the root filesystem outside of /usr/share and /var/www.
+# The former is used by web applications packaged in Debian,
+# the latter may be used for local directories served by the web server. If
+# your system is serving content from a sub-directory in /srv you must allow
+# access here, or in any related virtual host.
+<Directory />
+ Options FollowSymLinks
+ AllowOverride None
+ Require all denied
+</Directory>
+
+<Directory /usr/share>
+ AllowOverride None
+ Require all granted
+</Directory>
+
+<Directory /var/www/>
+ Options Indexes FollowSymLinks
+ AllowOverride All
+ Require all granted
+</Directory>
+
+#<Directory /srv/>
+# Options Indexes FollowSymLinks
+# AllowOverride None
+# Require all granted
+#</Directory>
+
+
+
+
+# AccessFileName: The name of the file to look for in each directory
+# for additional configuration directives. See also the AllowOverride
+# directive.
+#
+AccessFileName .htaccess
+
+#
+# The following lines prevent .htaccess and .htpasswd files from being
+# viewed by Web clients.
+#
+<FilesMatch "^\.ht">
+ Require all denied
+</FilesMatch>
+
+
+#
+# The following directives define some format nicknames for use with
+# a CustomLog directive.
+#
+# These deviate from the Common Log Format definitions in that they use %O
+# (the actual bytes sent including headers) instead of %b (the size of the
+# requested file), because the latter makes it impossible to detect partial
+# requests.
+#
+# Note that the use of %{X-Forwarded-For}i instead of %h is not recommended.
+# Use mod_remoteip instead.
+#
+LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined
+LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" combined
+LogFormat "%h %l %u %t \"%r\" %>s %O" common
+LogFormat "%{Referer}i -> %U" referer
+LogFormat "%{User-agent}i" agent
+
+# Include of directories ignores editors' and dpkg's backup files,
+# see README.Debian for details.
+
+# Include generic snippets of statements
+IncludeOptional conf-enabled/*.conf
+
+# Include the virtual host configurations:
+IncludeOptional sites-enabled/*.conf
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet


message: removed user-specific ssh key

diff:
ansible_connection: ssh
# ansible_ssh_user: centos
ansible_ssh_user: root
-ansible_ssh_private_key_file: /Users/supun/Desktop/shameera-key.pem
+#ansible_ssh_private_key_file: /Users/supun/Desktop/shameera-key.pem
# ansible_ssh_private_key_file: /Users/syodage/Projects/airavata-ansible/shameera-aws.pem.txt
user: centos
@@ -89,3 +89,4 @@ gfac_dir: "{{ deployment_dir }}/gfac-instance"
# Monitoring variables
monitor_email_address: "test.airavata@gmail.com"
monitor_email_password: "airavata"
+

original_message: removed user-specific ssh key


message: removed pga-test hosts

old_path: dev-tools/ansible/hosts

new_path: dev-tools/ansible/hosts

diff:
[pga]
149.165.170.103
-[pga-test]
-192.168.56.99 ansible_user=jecoulte ansible_become=true become_method=sudo ansible_ssh_user=jecoulte
-
[local]
localhost ansible_connection=local

diff:
- env_setup
- pga
-- hosts: pga-test
- tags: pga-test
- roles:
- - env_setup
- - pga
-
...

original_message: removed pga-test hosts


message:
added distro and greater version dependence
changed to using flags for distro and greater version to pull
in include files - turns out packages are named differently
between Ubuntu releases...

diff:
file: path="{{ doc_root_dir }}" state=directory
become: yes
-- include: install_deps_{{ ansible_os_family }}.yml
+- debug:
+ var: ansible_distribution_version
+
+- include: install_deps_{{ ansible_distribution }}_{{ ansible_distribution_major_version }}.yml
# - name: Check previous deployments
# stat: path="{{ doc_root_dir }}" get_md5=no get_checksum=no


message:
moving files to new distro/version name
moved include files to new scheme with distro
and major version instead of only os_family

change_type: RENAME

diff:
+---
+
+- name: Install prerequisites
+ apt: name="{{ item }}" state=latest update_cache=yes
+ with_items:
+ - git
+ - apache2
+ - libapache2-mod-php
+ - python-selinux
+ - python-apt
+ - unzip
+ - openssl
+ - curl
+ - php
+ - php-curl
+ - php-soap
+ - php-mysql
+ - php-mcrypt
+ - python-software-properties
+ become: yes
+
+- name: get python2-firewalld module...
+ unarchive:
+ src: https://github.com/hryamzik/python-firewalld/files/182007/python-firewalld_0.1-0.deb.zip
+ dest: /tmp/
+ remote_src: true
+
+- name: install python-firewalld
+ apt:
+ deb: /tmp/python-firewalld_0.1-0.deb
+#https://github.com/hryamzik/python-firewalld/files/182007/python-firewalld_0.1-0.deb.zip
+
+- name: install composer
+ shell: curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer
+ args:
+ creates: /usr/local/bin/composer
+ become: yes
+
+# this just creates symlinks in /etc/php5/{apache2,cli}/conf.d/20-mcryp.ini to /etc/php5/modules-avail/mcrypt.ini
+- name: enable mcrypt
+ command: phpenmod mcrypt
+ become: yes
+
+- name: enable apache rewrite
+ command: a2enmod rewrite
+ become: yes
+
+- name: create apache2 logdir
+ file:
+ path: /etc/apache2/logs
+ state: directory
+ owner: www-data
+ group: www-data
+ mode: 0775
+
+- name: set DocumentRoot in default site-enabled
+ lineinfile:
+ dest: /etc/apache2/sites-enabled/000-default.conf
+ line: ' DocumentRoot "/var/www/html/php-gateway/public"'
+ regexp: '^\s+DocumentRoot'
+ state: present


message:
added pga_repo var
changed the pga repo url to live in group_vars/all, since
this may need to be changed on a per-gateway basis - I'd
assume that individual gateway owners will change things like
the appearance, etc.

diff:
@@ -90,3 +90,5 @@ gfac_dir: "{{ deployment_dir }}/gfac-instance"
monitor_email_address: "test.airavata@gmail.com"
monitor_email_password: "airavata"
+# PGA variables
+pga_repo: "https://github.com/apache/airavata-php-gateway.git"

diff:
file: path="{{ doc_root_dir }}" state=directory
become: yes
-- debug:
- var: ansible_distribution_version
-
- include: install_deps_{{ ansible_distribution }}_{{ ansible_distribution_major_version }}.yml
# - name: Check previous deployments
# stat: path="{{ doc_root_dir }}" get_md5=no get_checksum=no
# register: check
-- name: delete previous deployments
- file: path="{{ doc_root_dir }}" state=absent
+# Why? if it's new, we just created this dir above...
+# if not, can just pull from the git repo in the next task...
+#- name: delete previous deployments
+# file: path="{{ doc_root_dir }}" state=absent
- name: Git clone php gateway
- git: repo=https://github.com/apache/airavata-php-gateway.git
- dest="{{ doc_root_dir }}" version="{{ git_branch }}"
+ git:
+ repo: "{{ pga_repo }}"
+ dest: "{{ doc_root_dir }}"
+ version: "{{ git_branch }}"
+ update: yes
+ force: yes
become: yes
- name: Create user data dir {{ user_data_dir }}


message:
name clarification; remove old commented tasks
clarified name for task to set selinux permissive, and removed
commented out tasks which are replaced by working versions.

diff:
# going to permissive from enforcing doesn't require a reboot
# permissive allows all, but still logs, allowing easy build
# of a working profile.
-- name: Disable selinux
+- name: set selinux permissive
selinux: state=permissive policy=default
become: yes
register: selinux_disable
- https
become: yes
-# - name: Edit file
-# lineinfile: dest="{{ httpd_conf_file }}" regexp="#\n\s*AllowOverride None" line="#\nAllowOverride All"
-# notify:
-# - restart httpd
-# become: yes
-
- name: copy httpd.conf file
template: src={{ httpd_name[ansible_os_family] }}.conf.j2 dest={{ httpd_conf_file_location[ansible_os_family] }}
become: yes
notify:
- restart httpd
-
-#- name: start httpd service
-# service: name=httpd state=started
-# become: yes


message:
Temporary fix for mysql-selinux issue
Tmp fix for this issue:
Adds selinux policy module to allow mysql to set uid/gid. Also
checks for .pp file which should be incuded in future releases
of MariaDB.

diff:
+module mysql-tmp 1.0;
+
+require {
+ type mysqld_safe_t;
+ class capability { setgid setuid };
+}
+
+#============= mysqld_safe_t ==============
+allow mysqld_safe_t self:capability { setgid setuid };

diff:
yum: name="{{ item }}" state=latest update_cache=yes
with_items: "{{ mysql_packages }}"
become_user: root
+#" for vim
+
+- name: check if mysql has been updated
+ stat: path=/usr/share/mysql/SELinux/mariadb.pp
+ register: mysql_selinux_update
+
+- debug:
+ var: mysql_selinux_update
+
+- name: double check policycoreutils installed
+ yum: name=policycoreutils-python state=installed
+ when: mysql_selinux_update.stat.exists == False
+
+- name: Copy SELinux type enforcement file
+ copy: src=mysql-tmp.te dest=/tmp/
+ when: mysql_selinux_update.stat.exists == False
+
+- name: Compile SELinux module file
+ command: checkmodule -M -m -o /tmp/mysql-tmp.mod /tmp/mysql-tmp.te
+ when: mysql_selinux_update.stat.exists == False
+
+- name: Build SELinux policy package
+ command: semodule_package -o /tmp/mysql-tmp.pp -m /tmp/mysql-tmp.mod
+ when: mysql_selinux_update.stat.exists == False
+
+#- name: check if mysql has been updated
+# stat: path=/tmp/mysql-tmp.pp
+# register: mysql_pp
+#
+#- debug:
+# var: mysql_pp
+
+- name: unLoad SELinux policy package
+ command: semodule -r mysql-tmp
+ when: mysql_selinux_update.stat.exists == False
+ become_user: root
+ ignore_errors: True
+
+- name: Load SELinux policy package
+ command: semodule -i /tmp/mysql-tmp.pp
+ when: mysql_selinux_update.stat.exists == False
+ become_user: root
+
+- name: Remove temporary files
+ file: path=/tmp/mysql-tmp.* state=absent
+ when: mysql_selinux_update.stat.exists == False
# TODO: SELinux issue for 10.0.29 MariaDB, need to allow setgid/setuid.
-# See https://www.rootusers.com/how-to-fix-mariadb-10-0-29-selinux-update-failure/
+## See https://www.rootusers.com/how-to-fix-mariadb-10-0-29-selinux-update-failure/
# and https://jira.mariadb.org/browse/MDEV-11789
# For now I'm not adding to Ansible script since this bug should be fixed in next MariaDB release
- name: start mariadb

original_message:
Temporary fix for mysql-selinux issue

Tmp fix for this issue: https://jira.mariadb.org/browse/MDEV-12231
Adds selinux policy module to allow mysql to set uid/gid. Also
checks for .pp file which should be incuded in future releases
of MariaDB.


message: Checking for duplicate gateway request

diff:
@@ -72,7 +72,9 @@ public class TenantProfileServiceHandler implements TenantProfileService.Iface {
@SecurityCheck
public String addGateway(AuthzToken authzToken, Gateway gateway) throws TenantProfileServiceException, AuthorizationException, TException {
try {
+ if (!checkDuplicate(gateway)) {
gateway = tenantProfileRepository.create(gateway);
+ }
if (gateway != null) {
logger.info("Added Airavata Gateway with Id: " + gateway.getGatewayId());
// replicate tenant at end-places only if status is APPROVED
@@ -183,7 +185,8 @@ public class TenantProfileServiceHandler implements TenantProfileService.Iface {
public boolean isGatewayExist(AuthzToken authzToken, String gatewayId) throws TenantProfileServiceException, AuthorizationException, TException {
try {
Gateway gateway = tenantProfileRepository.getGateway(gatewayId);
- return gateway != null;
+ boolean checkStatus = gateway.getGatewayApprovalStatus().equals(GatewayApprovalStatus.APPROVED);
+ return (gateway != null && checkStatus);
} catch (Exception ex) {
logger.error("Error checking if gateway-profile exists, reason: " + ex.getMessage(), ex);
TenantProfileServiceException exception = new TenantProfileServiceException();
@@ -191,4 +194,16 @@ public class TenantProfileServiceHandler implements TenantProfileService.Iface {
throw exception;
}
}
+
+ private boolean checkDuplicate(Gateway gateway) throws TenantProfileServiceException {
+ try {
+ Gateway duplicateGateway = tenantProfileRepository.getGateway(gateway.getGatewayId());
+ return ((duplicateGateway.getGatewayId() == gateway.getGatewayId()) && (duplicateGateway.getGatewayName() == gateway.getGatewayName()) && (duplicateGateway.getGatewayURL() == gateway.getGatewayURL()) && (duplicateGateway.getGatewayApprovalStatus().equals(GatewayApprovalStatus.APPROVED)));
+ } catch (Exception ex) {
+ logger.error("Error checking if duplicate gateway-profile exists, reason: " + ex.getMessage(), ex);
+ TenantProfileServiceException exception = new TenantProfileServiceException();
+ exception.setMessage("Error checking if duplicate gateway-profiles exists, reason: " + ex.getMessage());
+ throw exception;
+ }
+ }
}

original_message: Checking for duplicate gateway request


message: Modified TenantProfileRepository.java to pass paramenters to the query

diff:
@@ -200,7 +200,7 @@ public class TenantProfileServiceHandler implements TenantProfileService.Iface {
@SecurityCheck
public List<Gateway> getAllGatewaysForUser(AuthzToken authzToken, String requesterUsername) throws TenantProfileServiceException, AuthorizationException, TException {
try {
- return tenantProfileRepository.getAllGatewaysForUser();
+ return tenantProfileRepository.getAllGatewaysForUser(requesterUsername);
} catch (Exception ex) {
logger.error("Error getting user's gateway-profiles, reason: " + ex.getMessage(), ex);
TenantProfileServiceException exception = new TenantProfileServiceException();

diff:
package org.apache.airavata.service.profile.tenant.core.repositories;
import org.apache.airavata.model.workspace.Gateway;
+import org.apache.airavata.model.workspace.GatewayApprovalStatus;
import org.apache.airavata.service.profile.commons.repositories.AbstractRepository;
import org.apache.airavata.service.profile.commons.tenant.entities.GatewayEntity;
import org.apache.airavata.service.profile.commons.utils.QueryConstants;
@@ -48,6 +49,7 @@ public class TenantProfileRepository extends AbstractRepository<Gateway, Gateway
try {
Map<String, Object> queryParam = new HashMap<String, Object>();
queryParam.put(Gateway._Fields.GATEWAY_ID.getFieldName(), gatewayId);
+ queryParam.put(Gateway._Fields.GATEWAY_APPROVAL_STATUS.getFieldName(), GatewayApprovalStatus.APPROVED.name());
List<Gateway> gatewayList = select(QueryConstants.FIND_GATEWAY_BY_ID, 1, 0, queryParam);
if (!gatewayList.isEmpty()) {
gateway = gatewayList.get(0);
@@ -69,9 +71,11 @@ public class TenantProfileRepository extends AbstractRepository<Gateway, Gateway
}
}
- public List<Gateway> getAllGatewaysForUser () throws Exception {
+ public List<Gateway> getAllGatewaysForUser (String requesterUsername) throws Exception {
try {
- List<Gateway> gatewayList = select(QueryConstants.GET_USER_GATEWAYS);
+ Map<String, Object> queryParam = new HashMap<String, Object>();
+ queryParam.put(Gateway._Fields.REQUESTER_USERNAME.getFieldName(), requesterUsername);
+ List<Gateway> gatewayList = select(QueryConstants.GET_USER_GATEWAYS, 1, 0, queryParam);
return gatewayList;
} catch (Exception e){
logger.error("Error while getting the user's gateways, reason: ", e);


message: Modified the getAllAppModules() method

diff:
@@ -774,7 +774,8 @@ service RegistryService {
* Returns the list of all Application Module Objects.
*
*/
- list<application_deployment_model.ApplicationModule> getAllAppModules (1: required string gatewayId)
+ list<application_deployment_model.ApplicationModule> getAllAppModules (1: required string gatewayId
+ 2: required list<string> accessibleAppIds)
throws (1: registry_api_errors.RegistryServiceException rse)
/**

original_message: Modified the getAllAppModules() method


message: Changing permission check to READ

diff:
@@ -1768,7 +1768,7 @@ public class AiravataServerHandler implements Airavata.Iface {
ExperimentModel experiment = regClient.getExperiment(airavataExperimentId);
String userId = authzToken.getClaimsMap().get(Constants.USER_NAME);
String entityId = experiment.getExecutionId();
- if (!sharingClient.userHasAccess(gatewayId, userId + "@" + gatewayId, entityId,gatewayId + ":WRITE")) {
+ if (!sharingClient.userHasAccess(gatewayId, userId + "@" + gatewayId, entityId,gatewayId + ":READ")) {
logger.error(airavataExperimentId, "User does not have access to application module {}.", entityId);
throw new AuthorizationException("User does not have permission to access this resource");
}

original_message: Changing permission check to READ


message: Requested changes made (2)

diff:
@@ -378,11 +378,12 @@ public class ApplicationDeploymentImpl implements ApplicationDeployment {
}
@Override
- public List<ApplicationDeploymentDescription> getAllApplicationDeployements(String gatewayId) throws AppCatalogException {
+ public List<ApplicationDeploymentDescription> getAllApplicationDeployements(String gatewayId, List<String> accessibleAppIds) throws AppCatalogException {
List<ApplicationDeploymentDescription> deploymentDescriptions = new ArrayList<ApplicationDeploymentDescription>();
try {
AppDeploymentResource resource = new AppDeploymentResource();
resource.setGatewayId(gatewayId);
+ resource.setAccessibleAppIds(accessibleAppIds);
List<AppCatalogResource> resources = resource.getAll();
if (resources != null && !resources.isEmpty()){
deploymentDescriptions = AppCatalogThriftConversion.getAppDepDescList(resources);

diff:
@@ -323,11 +323,12 @@ public class ApplicationInterfaceImpl implements ApplicationInterface {
}
@Override
- public List<ApplicationModule> getAllApplicationModules(String gatewayId) throws AppCatalogException {
+ public List<ApplicationModule> getAllApplicationModules(String gatewayId, List<String> accessibleAppIds) throws AppCatalogException {
List<ApplicationModule> applicationModules = new ArrayList<ApplicationModule>();
try {
AppModuleResource resource = new AppModuleResource();
resource.setGatewayId(gatewayId);
+ resource.setAccessibleAppIds(accessibleAppIds);
List<AppCatalogResource> resources = resource.getAll();
if (resources != null && !resources.isEmpty()){
applicationModules = AppCatalogThriftConversion.getAppModules(resources);

diff:
* under the License.
*/
package org.apache.airavata.registry.core.app.catalog.resources;
+import java.util.ArrayList;
public abstract class AppCatAbstractResource implements AppCatalogResource {
// table names
@@ -203,6 +204,7 @@ public abstract class AppCatAbstractResource implements AppCatalogResource {
public static final String MODULE_NAME = "moduleName";
public static final String MODULE_VERSION = "moduleVersion";
public static final String MODULE_DESC = "moduleDesc";
+ public static final String ACCESSIBLE_APP_IDS = "accessibleAppIds";
}
public final class ApplicationDeploymentConstants {
@@ -214,6 +216,7 @@ public abstract class AppCatAbstractResource implements AppCatalogResource {
public static final String APPLICATION_DESC = "applicationDesc";
public static final String ENV_MODULE_LOAD_CMD = "envModuleLoaString";
public static final String PARALLELISM = "parallelism";
+ public static final String ACCESSIBLE_APP_IDS = "accessibleAppIds";
}
public final class LibraryPrepandPathConstants {

diff:
@@ -55,10 +55,13 @@ public class AppDeploymentResource extends AppCatAbstractResource {
private AppModuleResource moduleResource;
private Timestamp createdTime;
private Timestamp updatedTime;
+ private List<String> accessibleAppIds;
- public String getGatewayId() {
- return gatewayId;
- }
+ public List<String> getAccessibleAppIds() { return accessibleAppIds; }
+
+ public void setAccessibleAppIds(List<String> accessibleAppIds) { this.accessibleAppIds = accessibleAppIds; }
+
+ public String getGatewayId() { return gatewayId; }
public void setGatewayId(String gatewayId) {
this.gatewayId = gatewayId;
@@ -314,6 +317,7 @@ public class AppDeploymentResource extends AppCatAbstractResource {
em.getTransaction().begin();
AppCatalogQueryGenerator generator = new AppCatalogQueryGenerator(APPLICATION_DEPLOYMENT);
generator.setParameter(ApplicationDeploymentConstants.GATEWAY_ID, gatewayId);
+ generator.setParameter(ApplicationDeploymentConstants.ACCESSIBLE_APP_IDS, accessibleAppIds);
Query q = generator.selectQuery(em);
List results = q.getResultList();
if (results.size() != 0) {

diff:
@@ -44,6 +44,11 @@ public class AppModuleResource extends AppCatAbstractResource {
private Timestamp createdTime;
private Timestamp updatedTime;
private String gatewayId;
+ private List<String> accessibleAppIds;
+
+ public List<String> getAccessibleAppIds() { return accessibleAppIds; }
+
+ public void setAccessibleAppIds(List<String> accessibleAppIds) { this.accessibleAppIds = accessibleAppIds; }
public String getGatewayId() {
return gatewayId;
@@ -227,6 +232,7 @@ public class AppModuleResource extends AppCatAbstractResource {
em.getTransaction().begin();
AppCatalogQueryGenerator generator = new AppCatalogQueryGenerator(APPLICATION_MODULE);
generator.setParameter(ApplicationModuleConstants.GATEWAY_ID, gatewayId);
+ generator.setParameter(ApplicationModuleConstants.ACCESSIBLE_APP_IDS, accessibleAppIds);
Query q = generator.selectQuery(em);
List<?> results = q.getResultList();
for (Object result : results) {

diff:
@@ -21,6 +21,7 @@ package org.apache.airavata.registry.core.app.catalog.util;
import javax.persistence.EntityManager;
import javax.persistence.Query;
+import java.util.List;
import java.util.HashMap;
import java.util.Map;
@@ -74,7 +75,12 @@ public class AppCatalogQueryGenerator {
if (!matchString.equals("")){
matchString+=" AND ";
}
+ if (matches.get(colName) instanceof List){
+ matchString+=TABLE_OBJ+"."+colName+" IN :" + paramName;
+ }
+ else{
matchString+=TABLE_OBJ+"."+colName+" =:"+paramName;
+ }
paramCount++;
}
queryString+=" WHERE "+matchString;

original_message: Requested changes made (2)


message: Modified AppDeploymentTest.java

diff:
@@ -134,7 +134,9 @@ public class AppDeploymentTest {
List<String> allApplicationDeployementIds = appDep.getAllApplicationDeployementIds();
System.out.println("******** Size of all App deployments ids *********** : " + allApplicationDeployementIds.size());
- List<ApplicationDeploymentDescription> allApplicationDeployements = appDep.getAllApplicationDeployements(ServerSettings.getDefaultUserGateway());
+ List<String> accessibleAppIds = new ArrayList<>();
+ accessibleAppIds.add(wrfModuleId);
+ List<ApplicationDeploymentDescription> allApplicationDeployements = appDep.getAllApplicationDeployements(ServerSettings.getDefaultUserGateway(), accessibleAppIds);
System.out.println("******** Size of all App deployments *********** : " + allApplicationDeployements.size());
assertTrue("App interface saved successfully", app != null);

original_message: Modified AppDeploymentTest.java


message: Making interface changes

diff:
@@ -53,7 +53,20 @@ public interface ApplicationDeployment {
*/
List<ApplicationDeploymentDescription> getApplicationDeployements (Map<String, String> filters) throws AppCatalogException;
- List<ApplicationDeploymentDescription> getAllApplicationDeployements (String gatewayId, List<String> accessibleIds) throws AppCatalogException;
+ /**
+ * This method will return a list of all application deployments
+ * @param gatewayId Gateway ID
+ * @return list of all application deployments
+ */
+ List<ApplicationDeploymentDescription> getAllApplicationDeployements (String gatewayId) throws AppCatalogException;
+
+ /**
+ * This method will return a list of all application deployments
+ * @param gatewayId Gateway ID
+ * @param accessibleAppIds List of Accessible App IDs
+ * @return list of all application deployments accessible to the user
+ */
+ List<ApplicationDeploymentDescription> getAccessibleApplicationDeployements (String gatewayId, List<String> accessibleAppIds) throws AppCatalogException;
List<String> getAllApplicationDeployementIds () throws AppCatalogException;

diff:
@@ -86,7 +86,20 @@ public interface ApplicationInterface {
*/
List<ApplicationModule> getApplicationModules(Map<String, String> filters) throws AppCatalogException;
- List<ApplicationModule> getAllApplicationModules(String gatewayId, List<String> accessibleAppIds) throws AppCatalogException;
+ /**
+ * This method will return a list of all application modules
+ * @param gatewayId Gateway ID
+ * @return list of all application modules
+ */
+ List<ApplicationModule> getAllApplicationModules(String gatewayId) throws AppCatalogException;
+
+ /**
+ * This method will return a list of all application modules
+ * @param gatewayId Gateway ID
+ * @param accessibleAppIds List of Accessible App IDs accessible to the user
+ * @return list of all application modules accessible to the user
+ */
+ List<ApplicationModule> getAccessibleApplicationModules(String gatewayId, List<String> accessibleAppIds) throws AppCatalogException;
/**
* This method will return a list of application interfaces according to given search criteria

original_message: Making interface changes


message: Adding Migrator class to create everyone group in all domains

diff:
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.airavata.sharing.registry.migrator.airavata;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
+import org.apache.airavata.registry.api.RegistryService;
+import org.apache.airavata.registry.api.client.RegistryServiceClientFactory;
+import org.apache.airavata.sharing.registry.client.SharingRegistryServiceClientFactory;
+import org.apache.airavata.sharing.registry.models.*;
+import org.apache.airavata.sharing.registry.service.cpi.SharingRegistryService;
+import org.apache.thrift.TException;
+import java.util.*;
+
+public class EveryoneGroupMigrator {
+
+ public static void main(String[] args) throws ClassNotFoundException, TException, ApplicationSettingsException {
+
+ RegistryService.Client registryClient = RegistryServiceClientFactory.createRegistryClient("149.165.169.138",8970);
+ SharingRegistryService.Client sharingClient = SharingRegistryServiceClientFactory.createSharingRegistryClient("149.165.169.138", 7878);
+
+ List<Domain> domainList = sharingClient.getDomains(-1, 0);
+ for (Domain domain : domainList) {
+ String groupId = "everyone@" + domain.domainId;
+ if (!sharingClient.isGroupExists(domain.domainId, groupId)) {
+ UserGroup userGroup = new UserGroup();
+ userGroup.setGroupId(groupId);
+ userGroup.setDomainId(domain.domainId);
+ userGroup.setGroupCardinality(GroupCardinality.MULTI_USER);
+ userGroup.setCreatedTime(System.currentTimeMillis());
+ userGroup.setUpdatedTime(System.currentTimeMillis());
+ userGroup.setName("everyone");
+ userGroup.setDescription("Default Group");
+ userGroup.setGroupType(GroupType.DOMAIN_LEVEL_GROUP);
+ sharingClient.createGroup(userGroup);
+
+ List<User> userList = sharingClient.getUsers(domain.domainId, -1, 0);
+ List<String> users = new ArrayList<>();
+ for (User user : userList) {
+ users.add(user.getUserId());
+ }
+ sharingClient.addUsersToGroup(domain.domainId, users, groupId);
+
+ List<ApplicationDeploymentDescription> applicationDeploymentDescriptionList = registryClient.getAllApplicationDeployments(domain.domainId);
+ for (ApplicationDeploymentDescription applicationDeploymentDescription : applicationDeploymentDescriptionList) {
+ sharingClient.shareEntityWithGroups(domain.domainId, applicationDeploymentDescription.getAppDeploymentId(), Arrays.asList(groupId),domain.domainId+":READ", true);
+ }
+ }
+ }
+
+ }
+
+}

diff:
<artifactId>airavata-sharing-registry-stubs</artifactId>
<version>${project.parent.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>registry-api-stubs</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>airavata-server-configuration</artifactId>

diff:
@@ -277,7 +277,9 @@ public class SharingRegistryServerHandler implements SharingRegistryService.Ifac
group.setUpdatedTime(System.currentTimeMillis());
(new UserGroupRepository()).create(group);
+ if(group.ownerId != null)
addUsersToGroup(group.domainId, Arrays.asList(group.ownerId), group.groupId);
+
return group.groupId;
}catch (Throwable ex) {
logger.error(ex.getMessage(), ex);


message: Adding changes to entity creation for Application Deployments

diff:
@@ -172,10 +172,10 @@ public class AiravataServerHandler implements Airavata.Iface {
client.createEntityType(entityType);
entityType = new EntityType();
- entityType.setEntityTypeId(domain.domainId+":APPLICATION");
+ entityType.setEntityTypeId(domain.domainId+":APPLICATION-DEPLOYMENT");
entityType.setDomainId(domain.domainId);
- entityType.setName("APPLICATION");
- entityType.setDescription("Application entity type");
+ entityType.setName("APPLICATION-DEPLOYMENT");
+ entityType.setDescription("Application Deployment entity type");
client.createEntityType(entityType);
//Creating Permission Types for each domain
@@ -274,10 +274,10 @@ public class AiravataServerHandler implements Airavata.Iface {
sharingClient.createEntityType(entityType);
entityType = new EntityType();
- entityType.setEntityTypeId(domain.domainId+":APPLICATION");
+ entityType.setEntityTypeId(domain.domainId+":APPLICATION-DEPLOYMENT");
entityType.setDomainId(domain.domainId);
- entityType.setName("APPLICATION");
- entityType.setDescription("Application entity type");
+ entityType.setName("APPLICATION-DEPLOYMENT");
+ entityType.setDescription("Application Deployment entity type");
sharingClient.createEntityType(entityType);
//Creating Permission Types for each domain

diff:
@@ -38,6 +38,8 @@ public class AiravataDataMigrator {
SharingRegistryServerHandler sharingRegistryServerHandler = new SharingRegistryServerHandler();
+ String applicationDeploymentOwner = args[0];
+
String query = "SELECT * FROM GATEWAY";
Statement statement = expCatConnection.createStatement();
ResultSet rs = statement.executeQuery(query);
@@ -79,10 +81,10 @@ public class AiravataDataMigrator {
sharingRegistryServerHandler.createEntityType(entityType);
entityType = new EntityType();
- entityType.setEntityTypeId(domain.domainId+":APPLICATION");
+ entityType.setEntityTypeId(domain.domainId+":APPLICATION-DEPLOYMENT");
entityType.setDomainId(domain.domainId);
- entityType.setName("APPLICATION");
- entityType.setDescription("Application entity type");
+ entityType.setName("APPLICATION-DEPLOYMENT");
+ entityType.setDescription("Application Deployment entity type");
if (!sharingRegistryServerHandler.isEntityTypeExists(entityType.domainId, entityType.entityTypeId))
sharingRegistryServerHandler.createEntityType(entityType);
@@ -189,19 +191,19 @@ public class AiravataDataMigrator {
}
}
- //Creating application entries
- query = "SELECT * FROM APPLICATION";
+ //Creating application deployment entries
+ query = "SELECT * FROM APPLICATION_DEPLOYMENT";
statement = expCatConnection.createStatement();
rs = statement.executeQuery(query);
while(rs.next()){
try {
Entity entity = new Entity();
- entity.setEntityId(rs.getString("APPLICATION_ID"));
+ entity.setEntityId(rs.getString("DEPLOYMENT_ID"));
entity.setDomainId(rs.getString("GATEWAY_ID"));
- entity.setEntityTypeId(rs.getString("GATEWAY_ID") + ":APPLICATION");
- entity.setOwnerId(rs.getString("USER_NAME") + "@" + rs.getString("GATEWAY_ID"));
- entity.setName(rs.getString("APPLICATION_NAME"));
- entity.setDescription(rs.getString("DESCRIPTION"));
+ entity.setEntityTypeId(rs.getString("GATEWAY_ID") + ":APPLICATION-DEPLOYMENT");
+ entity.setOwnerId(applicationDeploymentOwner);
+ entity.setName(rs.getString("DEPLOYMENT_ID"));
+ entity.setDescription(rs.getString("APPLICATION_DESC"));
if(entity.getDescription() == null)
entity.setFullText(entity.getName());
else


message: Adding application deployments to everyone group

diff:
package org.apache.airavata.sharing.registry.migrator.airavata;
import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.sharing.registry.client.SharingRegistryServiceClientFactory;
import org.apache.airavata.sharing.registry.models.*;
import org.apache.airavata.sharing.registry.server.SharingRegistryServerHandler;
import org.apache.thrift.TException;
+//import org.apache.airavata.sharing.registry.service.cpi.SharingRegistryService;
+import java.util.Arrays;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -37,6 +40,7 @@ public class AiravataDataMigrator {
Connection expCatConnection = ConnectionFactory.getInstance().getExpCatConnection();
SharingRegistryServerHandler sharingRegistryServerHandler = new SharingRegistryServerHandler();
+ //SharingRegistryService.Client sharingClient = SharingRegistryServiceClientFactory.createSharingRegistryClient("149.165.169.138", 7878);
String applicationDeploymentOwner = args[0];
@@ -213,6 +217,9 @@ public class AiravataDataMigrator {
if (!sharingRegistryServerHandler.isEntityExists(entity.domainId, entity.entityId))
sharingRegistryServerHandler.createEntity(entity);
+ String groupId = "everyone@" + entity.domainId;
+ sharingRegistryServerHandler.shareEntityWithGroups(entity.domainId, entity.entityId, Arrays.asList(groupId), entity.domainId+":READ", true);
+ //sharingClient.shareEntityWithGroups(entity.domainId, entity.entityId, Arrays.asList(groupId), entity.domainId+":READ", true);
} catch (Exception ex){
ex.printStackTrace();
}


message: Modified createUser() to add new user to everyone group

diff:
@@ -36,6 +36,8 @@ public class EveryoneGroupMigrator {
RegistryService.Client registryClient = RegistryServiceClientFactory.createRegistryClient("149.165.169.138",8970);
SharingRegistryService.Client sharingClient = SharingRegistryServiceClientFactory.createSharingRegistryClient("149.165.169.138", 7878);
+ String groupOwner = args[0];
+
List<Domain> domainList = sharingClient.getDomains(-1, 0);
for (Domain domain : domainList) {
String groupId = "everyone@" + domain.domainId;
@@ -48,6 +50,7 @@ public class EveryoneGroupMigrator {
userGroup.setUpdatedTime(System.currentTimeMillis());
userGroup.setName("everyone");
userGroup.setDescription("Default Group");
+ userGroup.setOwnerId(groupOwner);
userGroup.setGroupType(GroupType.DOMAIN_LEVEL_GROUP);
sharingClient.createGroup(userGroup);

diff:
@@ -164,6 +164,9 @@ public class SharingRegistryServerHandler implements SharingRegistryService.Ifac
userGroup.setGroupCardinality(GroupCardinality.SINGLE_USER);
(new UserGroupRepository()).create(userGroup);
+ String groupId = "everyone@" + user.domainId;
+ addUsersToGroup(user.domainId, Arrays.asList(user.userId), groupId);
+
return user.userId;
}catch (Throwable ex) {
logger.error(ex.getMessage(), ex);
@@ -277,9 +280,7 @@ public class SharingRegistryServerHandler implements SharingRegistryService.Ifac
group.setUpdatedTime(System.currentTimeMillis());
(new UserGroupRepository()).create(group);
- if(group.ownerId != null)
addUsersToGroup(group.domainId, Arrays.asList(group.ownerId), group.groupId);
-
return group.groupId;
}catch (Throwable ex) {
logger.error(ex.getMessage(), ex);


message: Creating a default everyone group when adding a new gateway

diff:
@@ -299,6 +299,20 @@ public class AiravataServerHandler implements Airavata.Iface {
permissionType.setDescription("Write permission type");
sharingClient.createPermissionType(permissionType);
+ //Create an "everyone" group for the domain
+ String groupId = "everyone@" + domain.domainId;
+ UserGroup userGroup = new UserGroup();
+ userGroup.setGroupId(groupId);
+ userGroup.setDomainId(domain.domainId);
+ userGroup.setGroupCardinality(GroupCardinality.MULTI_USER);
+ userGroup.setCreatedTime(System.currentTimeMillis());
+ userGroup.setUpdatedTime(System.currentTimeMillis());
+ userGroup.setOwnerId(authzToken.getClaimsMap().get(Constants.USER_NAME));
+ userGroup.setName("everyone");
+ userGroup.setDescription("Default Group");
+ userGroup.setGroupType(GroupType.DOMAIN_LEVEL_GROUP);
+ sharingClient.createGroup(userGroup);
+
registryClientPool.returnResource(registryClient);
sharingClientPool.returnResource(sharingClient);

diff:
@@ -164,6 +164,7 @@ public class SharingRegistryServerHandler implements SharingRegistryService.Ifac
userGroup.setGroupCardinality(GroupCardinality.SINGLE_USER);
(new UserGroupRepository()).create(userGroup);
+ // Add user to the domain's "everyone" group
String groupId = "everyone@" + user.domainId;
addUsersToGroup(user.domainId, Arrays.asList(user.userId), groupId);


message: Corrected the everyone group's ownerID assignment

diff:
@@ -433,7 +433,7 @@ public class AiravataServerHandler implements Airavata.Iface {
userGroup.setGroupCardinality(GroupCardinality.MULTI_USER);
userGroup.setCreatedTime(System.currentTimeMillis());
userGroup.setUpdatedTime(System.currentTimeMillis());
- userGroup.setOwnerId(authzToken.getClaimsMap().get(Constants.USER_NAME));
+ userGroup.setOwnerId(authzToken.getClaimsMap().get(Constants.USER_NAME) + "@" + domain.domainId);
userGroup.setName("everyone");
userGroup.setDescription("Default Group");
userGroup.setGroupType(GroupType.DOMAIN_LEVEL_GROUP);


message: Adding registry-stubs dependency for the data migrator

diff:
<artifactId>airavata-credential-store-stubs</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>registry-api-stubs</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>services-security</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>


message: Modified Sharing Registry tests and Data Migrator

diff:
<artifactId>airavata-data-models</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>registry-api-service</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>airavata-credential-store-stubs</artifactId>
<artifactId>slf4j-api</artifactId>
<version>1.7.12</version>
</dependency>
+ <dependency>
+ <groupId>org.mariadb.jdbc</groupId>
+ <artifactId>mariadb-java-client</artifactId>
+ <version>2.0.2</version>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file

diff:
package org.apache.airavata.sharing.registry.migrator.airavata;
import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
import org.apache.airavata.model.group.ResourcePermissionType;
import org.apache.airavata.model.group.ResourceType;
-import org.apache.airavata.sharing.registry.client.SharingRegistryServiceClientFactory;
import org.apache.airavata.sharing.registry.models.*;
import org.apache.airavata.sharing.registry.server.SharingRegistryServerHandler;
import org.apache.thrift.TException;
-import org.apache.airavata.sharing.registry.service.cpi.SharingRegistryService;
import org.apache.airavata.credential.store.cpi.CredentialStoreService;
import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.credential.store.client.CredentialStoreClientFactory;
@@ -49,7 +48,6 @@ public class AiravataDataMigrator {
Connection expCatConnection = ConnectionFactory.getInstance().getExpCatConnection();
SharingRegistryServerHandler sharingRegistryServerHandler = new SharingRegistryServerHandler();
- SharingRegistryService.Client sharingClient = SharingRegistryServiceClientFactory.createSharingRegistryClient("149.165.169.138", 7878);
CredentialStoreService.Client credentialStoreServiceClient = getCredentialStoreServiceClient();
String query = "SELECT * FROM GATEWAY";
@@ -219,16 +217,41 @@ public class AiravataDataMigrator {
}
}
+ //Map to reuse the domain ID and owner for creating application-deployment entities
+ Map<String, String> domainOwnerMap = new HashMap<>();
+
//Creating the everyone group
- List<Domain> domainList = sharingClient.getDomains(-1, 0);
+ List<Domain> domainList = sharingRegistryServerHandler.getDomains(0, -1);
for (Domain domain : domainList) {
- GatewayResourceProfile gatewayResourceProfile = getRegistryServiceClient().getGatewayResourceProfile(domain.domainId);
+ GatewayResourceProfile gatewayResourceProfile = null;
+ try {
+ gatewayResourceProfile = getRegistryServiceClient().getGatewayResourceProfile(domain.domainId);
+ } catch (Exception e) {
+ System.out.println("Skipping creating everyone group for " + domain.domainId + " because it doesn't have a GatewayResourceProfile");
+ continue;
+ }
+ if (gatewayResourceProfile.getIdentityServerPwdCredToken() == null) {
+ System.out.println("Skipping creating everyone group for " + domain.domainId + " because it doesn't have an identity server pwd credential token");
+ continue;
+ }
+ String groupOwner = null;
+ try {
PasswordCredential credential = credentialStoreServiceClient.getPasswordCredential(
gatewayResourceProfile.getIdentityServerPwdCredToken(), gatewayResourceProfile.getGatewayID());
- String groupOwner = credential.getLoginUserName();
+ groupOwner = credential.getLoginUserName();
+ } catch (Exception e) {
+ System.out.println("Skipping creating everyone group for " + domain.domainId + " because the identity server pwd credential could not be retrieved.");
+ continue;
+ }
+ domainOwnerMap.put(domain.domainId, groupOwner);
String groupId = "everyone@" + domain.domainId;
- if (!sharingClient.isGroupExists(domain.domainId, groupId)) {
+ String ownerId = groupOwner + "@" + domain.domainId;
+ if (!sharingRegistryServerHandler.isUserExists(domain.domainId, ownerId)) {
+ System.out.println("Skipping creating everyone group for " + domain.domainId + " because admin user doesn't exist in sharing registry.");
+ continue;
+ }
+ if (!sharingRegistryServerHandler.isGroupExists(domain.domainId, groupId)) {
UserGroup userGroup = new UserGroup();
userGroup.setGroupId(groupId);
userGroup.setDomainId(domain.domainId);
@@ -237,55 +260,45 @@ public class AiravataDataMigrator {
userGroup.setUpdatedTime(System.currentTimeMillis());
userGroup.setName("everyone");
userGroup.setDescription("Default Group");
- userGroup.setOwnerId(groupOwner + "@" + domain.domainId);
+ userGroup.setOwnerId(ownerId);
userGroup.setGroupType(GroupType.DOMAIN_LEVEL_GROUP);
- sharingClient.createGroup(userGroup);
+ sharingRegistryServerHandler.createGroup(userGroup);
- List<User> userList = sharingClient.getUsers(domain.domainId, -1, 0);
+ List<User> userList = sharingRegistryServerHandler.getUsers(domain.domainId, 0, -1);
List<String> users = new ArrayList<>();
for (User user : userList) {
users.add(user.getUserId());
}
- sharingClient.addUsersToGroup(domain.domainId, users, groupId);
+ sharingRegistryServerHandler.addUsersToGroup(domain.domainId, users, groupId);
}
}
//Creating application deployment entries
- query = "SELECT * FROM APPLICATION_DEPLOYMENT";
- statement = expCatConnection.createStatement();
- rs = statement.executeQuery(query);
- while(rs.next()){
- try {
- GatewayResourceProfile gatewayResourceProfile = getRegistryServiceClient().
- getGatewayResourceProfile(rs.getString("GATEWAY_ID"));
- PasswordCredential credential = credentialStoreServiceClient.getPasswordCredential(
- gatewayResourceProfile.getIdentityServerPwdCredToken(), gatewayResourceProfile.getGatewayID());
- String applicationDeploymentOwner = credential.getLoginUserName();
-
+ for (String domainID : domainOwnerMap.keySet()) {
+ List<ApplicationDeploymentDescription> applicationDeploymentDescriptionList = getRegistryServiceClient().getAllApplicationDeployments(domainID);
+ for (ApplicationDeploymentDescription description : applicationDeploymentDescriptionList) {
Entity entity = new Entity();
- entity.setEntityId(rs.getString("DEPLOYMENT_ID"));
- entity.setDomainId(rs.getString("GATEWAY_ID"));
- entity.setEntityTypeId(rs.getString("GATEWAY_ID") + ":" + ResourceType.APPLICATION_DEPLOYMENT.name());
- entity.setOwnerId(applicationDeploymentOwner + "@" + rs.getString("GATEWAY_ID"));
- entity.setName(rs.getString("DEPLOYMENT_ID"));
- entity.setDescription(rs.getString("APPLICATION_DESC"));
+ entity.setEntityId(description.getAppDeploymentId());
+ entity.setDomainId(domainID);
+ entity.setEntityTypeId(entity.domainId + ":" + ResourceType.APPLICATION_DEPLOYMENT.name());
+ entity.setOwnerId(domainOwnerMap.get(domainID) + "@" + entity.domainId);
+ entity.setName(description.getAppDeploymentId());
+ entity.setDescription(description.getAppDeploymentDescription());
if (entity.getDescription() == null)
- entity.setFullText(entity.getName());
+ entity.setDescription(entity.getName());
else
entity.setFullText(entity.getName() + " " + entity.getDescription());
- Map<String, String> metadata = new HashMap<>();
- metadata.put("CREATION_TIME", rs.getDate("CREATION_TIME").toString());
if (!sharingRegistryServerHandler.isEntityExists(entity.domainId, entity.entityId))
sharingRegistryServerHandler.createEntity(entity);
- String groupId = "everyone@" + entity.domainId;
- sharingClient.shareEntityWithGroups(entity.domainId, entity.entityId, Arrays.asList(groupId), entity.domainId+":"+ ResourcePermissionType.EXEC, true);
- } catch (Exception ex){
- ex.printStackTrace();
+ String groupID = "everyone@" + entity.domainId;
+ sharingRegistryServerHandler.shareEntityWithGroups(entity.domainId, entity.entityId, Arrays.asList(groupID),
+ entity.domainId + ":" + ResourcePermissionType.EXEC, true);
}
}
expCatConnection.close();
+ System.out.println("Completed!");
}

diff:
@@ -157,7 +157,12 @@ public abstract class AbstractRepository<T, E, Id> {
entityManager.getTransaction().commit();
return r;
} finally {
+ if (entityManager.isOpen()) {
+ if (entityManager.getTransaction().isActive()) {
+ entityManager.getTransaction().rollback();
+ }
entityManager.close();
}
}
}
+}
\ No newline at end of file

diff:
@@ -166,7 +166,23 @@ public class SharingRegistryServerHandler implements SharingRegistryService.Ifac
// Add user to the domain's "everyone" group
String groupId = "everyone@" + user.domainId;
+ if (isGroupExists(user.domainId, groupId)) {
addUsersToGroup(user.domainId, Arrays.asList(user.userId), groupId);
+ }
+ else {
+ //Create an "everyone" group for the domain
+ UserGroup everyone = new UserGroup();
+ everyone.setGroupId(groupId);
+ everyone.setDomainId(user.domainId);
+ everyone.setGroupCardinality(GroupCardinality.MULTI_USER);
+ everyone.setCreatedTime(System.currentTimeMillis());
+ everyone.setUpdatedTime(System.currentTimeMillis());
+ everyone.setOwnerId(user.userId);
+ everyone.setName("everyone");
+ everyone.setDescription("Default Group");
+ everyone.setGroupType(GroupType.DOMAIN_LEVEL_GROUP);
+ createGroup(everyone);
+ }
return user.userId;
}catch (Throwable ex) {

diff:
@@ -59,7 +59,6 @@ public class SharingRegistryServerHandlerTest {
Assert.assertNotNull(sharingRegistryServerHandler.createDomain(domain));
Assert.assertTrue(sharingRegistryServerHandler.getDomains(0, 10).size() > 0);
-
//Creating users
User user1 = new User();
String userName1 = "test-user-1." + System.currentTimeMillis();
@@ -111,7 +110,7 @@ public class SharingRegistryServerHandlerTest {
userGroup1.setUpdatedTime(System.currentTimeMillis());
Assert.assertNotNull(sharingRegistryServerHandler.createGroup(userGroup1));
- Assert.assertTrue(sharingRegistryServerHandler.getAllMemberGroupsForUser(domainId, userId1).size() == 1);
+ Assert.assertTrue(sharingRegistryServerHandler.getAllMemberGroupsForUser(domainId, userId1).size() == 2);
UserGroup userGroup2 = new UserGroup();
String groupName2 = "test-group-2." + System.currentTimeMillis();
@@ -131,7 +130,7 @@ public class SharingRegistryServerHandlerTest {
sharingRegistryServerHandler.addUsersToGroup(domainId, Arrays.asList(userId1), groupId1);
sharingRegistryServerHandler.addUsersToGroup(domainId, Arrays.asList(userId2, userId3), groupId2);
- Assert.assertTrue(sharingRegistryServerHandler.getAllMemberGroupsForUser(domainId, userId3).size() == 1);
+ Assert.assertTrue(sharingRegistryServerHandler.getAllMemberGroupsForUser(domainId, userId3).size() == 2);
sharingRegistryServerHandler.addChildGroupsToParentGroup(domainId, Arrays.asList(groupId2), groupId1);
@@ -179,6 +178,17 @@ public class SharingRegistryServerHandlerTest {
String permissionTypeId2 = sharingRegistryServerHandler.createPermissionType(permissionType2);
Assert.assertNotNull(permissionTypeId2);
+ PermissionType permissionType3 = new PermissionType();
+ String permissionName3 = "EXEC";
+ permissionType3.setPermissionTypeId(domainId+":"+permissionName3);
+ permissionType3.setDomainId(domainId);
+ permissionType3.setName(permissionName3);
+ permissionType3.setDescription("EXEC description");
+ permissionType3.setCreatedTime(System.currentTimeMillis());
+ permissionType3.setUpdatedTime(System.currentTimeMillis());
+ String permissionTypeId3 = sharingRegistryServerHandler.createPermissionType(permissionType3);
+ Assert.assertNotNull(permissionTypeId3);
+
//Creating entity types
EntityType entityType1 = new EntityType();
String entityType1Name = "Project";
@@ -213,6 +223,17 @@ public class SharingRegistryServerHandlerTest {
String entityTypeId3 = sharingRegistryServerHandler.createEntityType(entityType3);
Assert.assertNotNull(entityTypeId3);
+ EntityType entityType4 = new EntityType();
+ String entityType4Name = "Application-Deployment";
+ entityType4.setEntityTypeId(domainId+":"+entityType4Name);
+ entityType4.setDomainId(domainId);
+ entityType4.setName(entityType4Name);
+ entityType4.setDescription("test entity type");
+ entityType4.setCreatedTime(System.currentTimeMillis());
+ entityType4.setUpdatedTime(System.currentTimeMillis());
+ String entityTypeId4 = sharingRegistryServerHandler.createEntityType(entityType4);
+ Assert.assertNotNull(entityTypeId4);
+
//Creating Entities
Entity entity1 = new Entity();
entity1.setEntityId(domainId+":Entity1");
@@ -281,6 +302,27 @@ public class SharingRegistryServerHandlerTest {
Assert.assertTrue(sharingRegistryServerHandler.userHasAccess(domainId, userId1, entityId4, permissionTypeId1));
Assert.assertFalse(sharingRegistryServerHandler.userHasAccess(domainId, userId3, entityId1, permissionTypeId1));
+ Entity entity5 = new Entity();
+ entity5.setEntityId(domainId+":Entity5");
+ entity5.setDomainId(domainId);
+ entity5.setEntityTypeId(entityTypeId4);
+ entity5.setOwnerId(userId1);
+ entity5.setName("App deployment name");
+ entity5.setDescription("App deployment description");
+ entity5.setFullText("App Deployment name app deployment description");
+ entity5.setCreatedTime(System.currentTimeMillis());
+ entity5.setUpdatedTime(System.currentTimeMillis());
+
+ String entityId5 = sharingRegistryServerHandler.createEntity(entity5);
+ Assert.assertNotNull(entityId5);
+
+ sharingRegistryServerHandler.shareEntityWithUsers(domainId, entityId5, Arrays.asList(userId2), permissionTypeId1, true);
+ sharingRegistryServerHandler.shareEntityWithGroups(domainId, entityId5, Arrays.asList(groupId2), permissionTypeId3, true);
+
+ Assert.assertTrue(sharingRegistryServerHandler.userHasAccess(domainId, userId3, entityId5, permissionTypeId3));
+ Assert.assertTrue(sharingRegistryServerHandler.userHasAccess(domainId, userId2, entityId5, permissionTypeId1));
+ Assert.assertFalse(sharingRegistryServerHandler.userHasAccess(domainId, userId3, entityId5, permissionTypeId2));
+
ArrayList<SearchCriteria> filters = new ArrayList<>();
SearchCriteria searchCriteria = new SearchCriteria();
searchCriteria.setSearchCondition(SearchCondition.LIKE);
@@ -301,7 +343,5 @@ public class SharingRegistryServerHandlerTest {
Assert.assertTrue(sharingRegistryServerHandler.getListOfSharedUsers(domainId, entityId1, domainId + ":OWNER").size()==1);
-// sharingRegistryServerHandler.revokeEntitySharingFromUsers(entityId1, Arrays.asList(userId2), permissionTypeId1);
-// sharingRegistryServerHandler.revokeEntitySharingFromGroups(entityId3, Arrays.asList(groupId2), permissionTypeId1);
}
}
\ No newline at end of file

diff:
@@ -313,7 +313,7 @@ public class SharingRegistryServiceTest {
searchCriteria.setValue("1");
searchCriteria.setSearchCondition(SearchCondition.GTE);
filters.add(searchCriteria);
- Assert.assertTrue(sharingServiceClient.searchEntities(domainId, "test-user-2", filters, 0, -1).size() == 1);
+ Assert.assertEquals(1, sharingServiceClient.searchEntities(domainId, "test-user-2", filters, 0, -1).size());
sharingServiceClient.revokeEntitySharingFromUsers(domainId, "test-project-1", Arrays.asList("test-user-2"), "WRITE");


message: Removing unwanted dependency

diff:
<artifactId>airavata-data-models</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>registry-api-service</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>airavata-credential-store-stubs</artifactId>

original_message: Removing unwanted dependency


message: Adding tilaks to Airavata pom.xml

diff:
<role>committer</role>
</roles>
</developer>
+ <developer>
+ <id>tilaks</id>
+ <name>Sneha Tilak</name>
+ <email>tilaks@apache.org</email>
+ <timezone>-5</timezone>
+ <organization>Indiana University</organization>
+ <organizationUrl>http://iub.edu</organizationUrl>
+ <roles>
+ <role>committer</role>
+ </roles>
+ </developer>
</developers>
<dependencyManagement>

original_message: Adding tilaks to Airavata pom.xml