blob: d01e3bafae1274c67e9a23dae7194cb65f1894cc [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.ignite.ci.runners;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.time.Duration;
import java.util.Iterator;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
import javax.cache.Cache;
import javax.xml.bind.JAXBException;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.ci.db.TcHelperDb;
import org.apache.ignite.ci.issue.Issue;
import org.apache.ignite.ci.issue.IssueKey;
import org.apache.ignite.tcbot.engine.issue.IssueType;
import org.apache.ignite.tcbot.engine.issue.IssuesStorage;
import org.apache.ignite.tcbot.engine.user.UserAndSessionsStorage;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.user.TcHelperUser;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.logger.slf4j.Slf4jLogger;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.tcbot.persistence.IgniteStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.tcservice.model.hist.BuildRef;
import org.apache.ignite.tcservice.model.result.Build;
import org.apache.ignite.tcservice.util.XmlUtil;
import org.jetbrains.annotations.NotNull;
import static org.apache.ignite.tcignited.history.BuildStartTimeStorage.BUILD_START_TIME_CACHE_NAME;
/**
* Utility class for connecting to a remote server.
*/
public class RemoteClientTmpHelper {
public static String DUMPS = "dumps";
private static boolean dumpDict = false;
/**
* @param args Args.
*/
public static void main(String[] args) {
mainSetUserAsAdmin(args);
//mainDumpAllUsers(args);
// mainExport(args);
// mainDropInvalidIssues(args);
System.err.println("Please insert option of main");
}
/**
* @param args Args.
*/
public static void mainSetUserAsAdmin(String[] args) {
try (Ignite ignite = tcbotServerConnectedClient()) {
IgniteCache<String, TcHelperUser> users = ignite.cache(UserAndSessionsStorage.USERS);
TcHelperUser user = users.get("dpavlov");
user.setAdmin(true);
users.put(user.username, user);
}
}
public static void mainDropInvalidIssues(String[] args) {
try (Ignite ignite = tcbotServerConnectedClient()) {
IgniteCache<IssueKey, Issue> bst = ignite.cache(IssuesStorage.BOT_DETECTED_ISSUES);
Iterator<Cache.Entry<IssueKey, Issue>> iter = bst.iterator();
try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dumpsDir(),
"Issues_dropped.txt")))) {
while (iter.hasNext()) {
Cache.Entry<IssueKey, Issue> next = iter.next();
boolean rmv = false;
Issue val = next.getValue();
if (val.type == null)
rmv = true;
//don't touch it
if (Objects.equals(val.type, IssueType.newContributedTestFailure.code()))
continue;
long ageDays = -1;
if (val != null && val.buildStartTs == null) {
if (val.detectedTs == null)
rmv = true;
else
ageDays = Duration.ofMillis(System.currentTimeMillis() - val.detectedTs).toDays();
rmv = true;
}
if(rmv) {
bst.remove(next.getKey());
String str = "Removing issue " + next.getKey() + " " + val.type + " detected " +
ageDays + " days ago\n";
writer.write(str);
System.err.println(str);
}
}
writer.flush();
}
catch (IOException e) {
throw new UncheckedIOException(e);
}
dumpDictionary(ignite);
}
}
public static void mainDumpIssues(String[] args) {
try (Ignite ignite = tcbotServerConnectedClient()) {
IgniteCache<IssueKey, Issue> bst = ignite.cache(IssuesStorage.BOT_DETECTED_ISSUES);
Iterator<Cache.Entry<IssueKey, Issue>> iter = bst.iterator();
try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dumpsDir(),
"Issues.txt")))) {
while (iter.hasNext()) {
Cache.Entry<IssueKey, Issue> next = iter.next();
Issue val = next.getValue();
long ageDays = -1;
if (val != null && val.buildStartTs != null)
ageDays = Duration.ofMillis(System.currentTimeMillis() - val.buildStartTs).toDays();
writer.write(next.getKey() + " " + val + " " +
ageDays + "\n");
}
writer.flush();
}
catch (IOException e) {
throw new UncheckedIOException(e);
}
dumpDictionary(ignite);
}
}
public static void mainDumpFatBuildStartTime(String[] args) {
try (Ignite ignite = tcbotServerConnectedClient()) {
IgniteCache<Long, FatBuildCompacted> bst = ignite.cache(FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME);
Iterator<Cache.Entry<Long, FatBuildCompacted>> iterator = bst.iterator();
try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dumpsDir(),
"fatBuildStartTime.txt")))) {
while (iterator.hasNext()) {
Cache.Entry<Long, FatBuildCompacted> next = iterator.next();
FatBuildCompacted val = next.getValue();
long ageDays = -1;
long startDateTs = -2;
if (val != null) {
startDateTs = val.getStartDateTs();
ageDays = Duration.ofMillis(System.currentTimeMillis() - startDateTs).toDays();
}
writer.write(next.getKey() + " " + startDateTs + " " +
ageDays + "\n");
}
}
catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}
public static void mainDumpBuildStartTime(String[] args) {
try (Ignite ignite = tcbotServerConnectedClient()) {
IgniteCache<Long, Long> bst = ignite.cache(BUILD_START_TIME_CACHE_NAME);
Iterator<Cache.Entry<Long, Long>> iterator = bst.iterator();
try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dumpsDir(),
"BuildStartTime.txt")))) {
while (iterator.hasNext()) {
Cache.Entry<Long, Long> next = iterator.next();
Long val = next.getValue();
long ageDays = -1;
if(val!=null)
ageDays = Duration.ofMillis(System.currentTimeMillis() - val).toDays();
writer.write(next.getKey() + " " + val + " " +
ageDays +"\n");
}
}
catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}
/**
* @param args Args.
*/
public static void mainResetUser(String[] args) {
try (Ignite ignite = tcbotServerConnectedClient()) {
IgniteCache<String, TcHelperUser> users = ignite.cache(UserAndSessionsStorage.USERS);
TcHelperUser user = users.get("user");
user.resetCredentials();
users.put(user.username, user);
}
}
/**
* @param args Args.
*/
public static void mainDestroyAllFatBuilds(String[] args) {
int fatBuildsCnt;
try (Ignite ignite = tcbotServerConnectedClient()) {
IgniteCache<Object, Object> cacheHistEntries = ignite.cache( FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME);
fatBuildsCnt = cacheHistEntries.size();
System.err.println("Start destroy() operation for fat builds: " + fatBuildsCnt);
cacheHistEntries.destroy();
System.err.println("Finish destroy() operation");
}
System.err.println("Test hist entries destroyed: [" + fatBuildsCnt + "] builds");
}
public static void mainDestroyTestHist(String[] args) {
int buildStartTimes;
try (Ignite ignite = tcbotServerConnectedClient()) {
IgniteCache<Object, Object> cacheStartTimes = ignite.cache(BUILD_START_TIME_CACHE_NAME);
buildStartTimes = cacheStartTimes.size();
System.err.println("Start destroy operation for build start times flag");
cacheStartTimes.destroy();
System.err.println("Finish destroy operation");
}
System.err.println("Test build start times destroyed [" + buildStartTimes + "] builds");
}
/**
* @param args Args.
*/
public static void main2(String[] args) {
int inconsistent;
try (Ignite ignite = tcbotServerConnectedClient()) {
inconsistent = validateBuildIdConsistency(ignite);
}
System.err.println("Inconsistent builds in queue found [" + inconsistent + "]");
}
public static int validateBuildIdConsistency(Ignite ignite) {
AtomicInteger inconsistent = new AtomicInteger();
String apacheSrvName = "apache";
int apache = ITeamcityIgnited.serverIdToInt(apacheSrvName);
IgniteCache<String, org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity> strings = ignite.cache(IgniteStringCompactor.STRINGS_CACHE);
org.apache.ignite.ci.teamcity.ignited.IgniteStringCompactor.CompactorEntity queuedEnt = strings.get(BuildRef.STATE_QUEUED);
int stateQ = queuedEnt.id();
IgniteCache<Long, BuildRefCompacted> cacheRef = ignite.cache(BuildRefDao.TEAMCITY_BUILD_CACHE_NAME);
IgniteCache<Long, FatBuildCompacted> cacheFat = ignite.cache(FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME);
cacheRef.forEach(
entry -> {
BuildRefCompacted buildRef = entry.getValue();
int buildId = BuildRefDao.cacheKeyToBuildId(entry.getKey());
if (buildRef.state() == stateQ && BuildRefDao.isKeyForServer(entry.getKey(), apache)) {
FatBuildCompacted fat = cacheFat.get(FatBuildDao.buildIdToCacheKey(apache, buildId));
if (fat != null && fat.getId() != buildId) {
dumpBuildRef(buildId, buildRef);
dumpFatBuild(cacheFat, apache, buildId);
inconsistent.incrementAndGet();
if (!fat.isOutdatedEntityVersion())
Preconditions.checkState(false);
}
}
}
);
return inconsistent.get();
}
public static void dumpOldBuild(int buildId, String href, Build fatBuild) {
try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dumpsDir(), "BuildOld" + buildId + ".txt")))) {
writer.write("<!--" + href + "-->\n");
writer.write(XmlUtil.save(fatBuild));
}
catch (IOException e) {
throw new UncheckedIOException(e);
}
catch (JAXBException e) {
e.printStackTrace();
}
}
private static void setupDisco(IgniteConfiguration cfg) {
final TcpDiscoverySpi spi = new TcpDiscoverySpi();
final int locPort = 54433;
spi.setLocalPort(locPort);
spi.setLocalPortRange(1);
TcpDiscoveryVmIpFinder finder = new TcpDiscoveryVmIpFinder();
finder.setAddresses(Lists.newArrayList("172.25.5.21:" + locPort));
spi.setIpFinder(finder);
cfg.setDiscoverySpi(spi);
}
public static void mainExport(String[] args) {
final Ignite ignite = tcbotServerConnectedClient();
if (dumpDict)
dumpDictionary(ignite);
if (true) {
IgniteCache<Long, FatBuildCompacted> cache1 = ignite.cache(FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME);
int apache = ITeamcityIgnited.serverIdToInt("apache");
int id = 4466392;
int id1 = 4465532;
dumpFatBuild(cache1, apache, id);
dumpFatBuild(cache1, apache, id1);
IgniteCache<Long, BuildRefCompacted> cache2 = ignite.cache(BuildRefDao.TEAMCITY_BUILD_CACHE_NAME);
dumpBuildRef(cache2, apache, id);
dumpBuildRef(cache2, apache, id1);
}
ignite.close();
}
public static void dumpDictionary(Ignite ignite) {
IgniteCache<String, Object> strings = ignite.cache(IgniteStringCompactor.STRINGS_CACHE);
try {
try (FileWriter writer = new FileWriter(new File(dumpsDir(), "Dictionary.txt"))) {
for (Cache.Entry<String, Object> next1 : strings) {
writer.write(next1.getValue().toString()
+ "\n");
}
}
}
catch (IOException e) {
throw new UncheckedIOException(e);
}
}
public static Ignite tcbotServerConnectedClient() {
final IgniteConfiguration cfg = new IgniteConfiguration();
setupDisco(cfg);
cfg.setGridLogger(new Slf4jLogger());
cfg.setClientMode(true);
return Ignition.start(cfg);
}
public static void dumpBuildRef(IgniteCache<Long, BuildRefCompacted> cache, int apache, int id) {
long l = BuildRefDao.buildIdToCacheKey(apache, id);
BuildRefCompacted compacted = cache.get(l);
dumpBuildRef(id, compacted);
}
public static void dumpBuildRef(int id, BuildRefCompacted compacted) {
Preconditions.checkNotNull(compacted, "Can't find build by ID " + id);
File dumps = dumpsDir();
try (FileWriter writer = new FileWriter(new File(dumps, "BuildRef " + id + ".txt"))) {
writer.write(compacted.toString());
}
catch (IOException e) {
throw new UncheckedIOException(e);
}
}
public static void mainDumpAllUsers(String[] args) {
try (Ignite ignite = tcbotServerConnectedClient()) {
IgniteCache<String, TcHelperUser>users = ignite.cache(UserAndSessionsStorage.USERS);
Iterator<Cache.Entry<String, TcHelperUser>> iterator = users.iterator();
while (iterator.hasNext()) {
Cache.Entry<String, TcHelperUser> next = iterator.next();
dumpUser(next.getValue());
}
}
}
private static void dumpUser(TcHelperUser value) {
try (FileWriter writer = new FileWriter(new File(dumpsDir(), "Build " + value.username() + ".txt"))) {
writer.write(value.toString());
}
catch (IOException e) {
throw new UncheckedIOException(e);
}
}
@NotNull public static File dumpsDir() {
File dumps = new File(DUMPS);
if (!dumps.exists())
Preconditions.checkState(dumps.mkdirs());
return dumps;
}
public static FatBuildCompacted dumpFatBuild(IgniteCache<Long, FatBuildCompacted> cache, int apache, int id) {
long l = FatBuildDao.buildIdToCacheKey(apache, id);
FatBuildCompacted compacted = cache.get(l);
Preconditions.checkNotNull(compacted, "Can't find build by ID " + id);
try (FileWriter writer = new FileWriter(new File(dumpsDir(), "Build " + id + ".txt"))) {
writer.write(compacted.toString());
}
catch (IOException e) {
throw new UncheckedIOException(e);
}
return compacted;
}
}