lzhengning
/
JittorMirror
mirror of https://github.com/Jittor/Jittor
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix windows encoding bugs

This commit is contained in:
Dun Liang 2022-03-09 18:23:42 +08:00
parent 53b377ee7d
commit 495d78ad20
16 changed files with 113 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
python/jittor/__init__.py
View File

@ -9,7 +9,7 @@
# file 'LICENSE.txt', which is part of this source code package.
# ***************************************************************
__version__ = '1.3.1.44'
__version__ = '1.3.1.45'
from jittor_utils import lock
with lock.lock_scope():
ori_int = int
@ -410,9 +410,7 @@ def flatten(input, start_dim=0, end_dim=-1):
return input.reshape(out_shape)
Var.flatten = flatten
def start_grad(x):
return x._update(x)
Var.detach_inplace = Var.start_grad = start_grad
Var.detach_inplace = Var.start_grad
def detach(x):
return x.detach()

5
python/jittor/compile_extern.py
View File

@ -190,6 +190,11 @@ def setup_cub():
def setup_cuda_extern():
if not has_cuda: return
check_ld_path = os.environ.get("LD_LIBRARY_PATH", "")
if "cuda" in check_ld_path.lower() and "lib" in check_ld_path.lower():
LOG.w(f"CUDA related path found in LD_LIBRARY_PATH({check_ld_path}), "
"This path may cause jittor found the wrong libs, "
"please unset LD_LIBRARY_PATH. ")
LOG.vv("setup cuda extern...")
cache_path_cuda = os.path.join(cache_path, "cuda")
cuda_include = os.path.join(jittor_path, "extern", "cuda", "inc")

18
python/jittor/compiler.py
View File

@ -199,7 +199,7 @@ def gen_jit_tests():
}} // jittor
"""
LOG.vvvv(jit_src)
with open(os.path.join(cache_path, "gen", "jit_tests.h"), 'w') as f:
with open(os.path.join(cache_path, "gen", "jit_tests.h"), 'w', encoding='utf8') as f:
f.write(jit_src)
def gen_jit_flags():
@ -257,7 +257,7 @@ def gen_jit_flags():
}} // jittor
"""
LOG.vvvv(jit_src)
with open(os.path.join(cache_path, "gen", "jit_flags.h"), 'w') as f:
with open(os.path.join(cache_path, "gen", "jit_flags.h"), 'w', encoding='utf8') as f:
f.write(jit_src)
def gen_jit_op_maker(op_headers, export=False, extra_flags=""):
@ -639,9 +639,9 @@ def compile_custom_op(header, source, op_name, warp=True):
make_cache_dir(cops_dir)
hname = os.path.join(cops_dir, op_name+"_op.h")
ccname = os.path.join(cops_dir, op_name+"_op.cc")
with open(hname, 'w') as f:
with open(hname, 'w', encoding='utf8') as f:
f.write(header)
with open(ccname, 'w') as f:
with open(ccname, 'w', encoding='utf8') as f:
f.write(source)
m = compile_custom_ops([hname, ccname])
return getattr(m, op_name)
@ -679,7 +679,7 @@ def compile_custom_ops(
dirname = os.path.dirname(name)
if dirname.endswith("inc"):
includes.append(dirname)
with open(name, "r") as f:
with open(name, "r", encoding='utf8') as f:
if "@pyjt" in f.read():
pyjt_includes.append(name)
bname = os.path.basename(name)
@ -736,7 +736,7 @@ def compile_custom_ops(
"init_module(PyModuleDef* mdef, PyObject* m) {",
f"jittor::pyjt_def_{bname}(m);")
with open(gen_head_fname, "w") as f:
with open(gen_head_fname, "w", encoding='utf8') as f:
f.write(gen_src)
LOG.vvv(f"Build custum ops lib:{gen_lib}")
@ -781,7 +781,7 @@ def compile_extern():
files = os.listdir(jittor_path_llvm)
# test_pass.cc is used for test link problem of llvm pass plugin
test_pass_path = os.path.join(cache_path_llvm, "test_pass.cc")
with open(test_pass_path, 'w') as f:
with open(test_pass_path, 'w', encoding='utf8') as f:
f.write("int main() {return 0;}")
# -fno-rtti fix link error
@ -1082,7 +1082,7 @@ if os.name == 'nt':
cc_flags = cc_flags.replace("-lstdc++", "")
cc_flags = cc_flags.replace("-ldl", "")
cc_flags += f" -L\"{py3_link_path}\" -lpython3{sys.version_info.minor} "
cc_flags += " -EHa -MD "
cc_flags += " -EHa -MD -utf-8 "
import jittor_utils
if jittor_utils.msvc_path:
mp = jittor_utils.msvc_path
@ -1217,7 +1217,7 @@ gen_jit_tests()
op_headers = glob.glob(jittor_path+"/src/ops/**/*op.h", recursive=True)
jit_src = gen_jit_op_maker(op_headers)
LOG.vvvv(jit_src)
with open(os.path.join(cache_path, "gen", "jit_op_maker.h"), 'w') as f:
with open(os.path.join(cache_path, "gen", "jit_op_maker.h"), 'w', encoding='utf8') as f:
f.write(jit_src)
cc_flags += f' -I\"{cache_path}\" -L\"{cache_path}\" -L\"{jit_utils.cache_path}\" '
# gen pyjt

2
python/jittor/notebook/60分钟快速入门Jittor/计图入门教程 2 --- 如何训练一个简单线性回归.ipynb
View File

@ -460,7 +460,7 @@
" train_loss_list.append(train_loss)\n",
" # 在验证集上进行验证，模型参数不做更新。\n",
" val_loss = val(model, x_val_var, y_val_var, loss_function)\n",
" val_loss_list.append(val_loss)\n",
" val_loss_list.append(val_loss.item())\n",
" \n",
"# 打印训练结束后的模型参数\n",
"print(\"After training: \\n\", model.state_dict())"

14
python/jittor/notebook/60分钟快速入门Jittor/计图入门教程 3 --- 尝试解决一个实际问题.ipynb
View File

@ -192,7 +192,7 @@
" \n",
" plt.imshow(inputs[num].numpy().transpose(1, 2, 0)) # 利用 matplotlib 根据第一个 input 绘制手写数字的图像\n",
" plt.show() # 展示图像\n",
" print(\"target:\", targets[num].data[0]) # 打印第一个 input 数据的真实标签值，即手写数字图像所表达的真实数字\n",
" print(\"target:\", targets[num].numpy()[0]) # 打印第一个 input 数据的真实标签值，即手写数字图像所表达的真实数字\n",
" break"
]
},
@ -910,12 +910,12 @@
" outputs = model(inputs) # 通过模型预测手写数字。outputs 中每个数据输出有 10 个分量，对应十个数字的相似度\n",
" loss = loss_function(outputs, targets) # 计算损失函数\n",
" optimizer.step(loss) # 根据损失函数，对模型参数进行优化、更新\n",
" train_losses.append(loss) # 记录该批次的 Loss\n",
" train_losses.append(loss.item()) # 记录该批次的 Loss\n",
" \n",
" if batch_idx % 10 == 0: # 每十个批次，打印一次训练集上的 Loss \n",
" print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
" epoch, batch_idx, len(train_loader),\n",
" 100. * batch_idx / len(train_loader), loss.data[0]))\n",
" 100. * batch_idx / len(train_loader), loss.item()))\n",
" return train_losses # 返回本纪元的 Loss\n",
"\n",
"\n",
@ -926,8 +926,8 @@
" total_num = 0 # 本纪元数据总数\n",
" for batch_idx, (inputs, targets) in enumerate(val_loader): # 通过测试集加载器，按批次迭代数据\n",
" outputs = model(inputs) # 通过模型预测手写数字。outputs 中每个数据输出有 10 个分量，对应十个数字的相似度\n",
" pred = np.argmax(outputs.data, axis=1) # 根据 10 个分量，选择最大相似度的为预测的数字值\n",
" correct = np.sum(targets.data==pred) # 计算本批次中，正确预测的次数，即数据标签等于预测值的数目\n",
" pred = np.argmax(outputs.numpy(), axis=1) # 根据 10 个分量，选择最大相似度的为预测的数字值\n",
" correct = np.sum(targets.numpy()==pred) # 计算本批次中，正确预测的次数，即数据标签等于预测值的数目\n",
" batch_size = inputs.shape[0] # 计算本批次中，数据的总数目\n",
" acc = correct / batch_size # 计算本批次的正确率\n",
" \n",
@ -1075,10 +1075,10 @@
" plt.imshow(inputs[num].numpy().transpose(1, 2, 0)) # 绘制该数据的手写数字图像\n",
" plt.show() \n",
" \n",
" print(\"target:\", targets[num].data[0]) # 打印该数据的真实标签值\n",
" print(\"target:\", targets[num].numpy()[0]) # 打印该数据的真实标签值\n",
" \n",
" outputs = model(inputs) # 模型根据输入数据进行预测\n",
" pred = np.argmax(outputs.data, axis=1) # 根据最大相似度得到预测值\n",
" pred = np.argmax(outputs.numpy(), axis=1) # 根据最大相似度得到预测值\n",
" print(\"prediction:\", pred[num]) # 打印该数据的预测值\n",
" break"
]

2
python/jittor/notebook/md_to_ipynb.py
View File

@ -63,5 +63,5 @@ for mdname in all_md:
ipynb_name = os.path.basename(mdname[:-2])+"ipynb"
ipynb_name = os.path.join(notebook_dir, ipynb_name)
print(mdname, len(src), len(blocks), len(cells), "--->", ipynb_name)
with open(ipynb_name, "w") as f:
with open(ipynb_name, "w", encoding='utf8') as f:
f.write(json.dumps(ipynb))

10
python/jittor/pyjt_compiler.py
View File

@ -858,13 +858,13 @@ def compile_src(src, h, basename):
def compile_single(head_file_name, src_file_name, src=None):
basename = os.path.basename(head_file_name).split(".")[0]
if src==None:
with open(head_file_name, 'r') as f:
with open(head_file_name, 'r', encoding='utf8') as f:
src = f.read()
code = compile_src(src, head_file_name, basename)
if not code: return False
LOG.vvv("write to", src_file_name)
LOG.vvvv(code)
with open(src_file_name, 'w') as f:
with open(src_file_name, 'w', encoding='utf8') as f:
f.write(code)
return True
@ -875,14 +875,14 @@ def compile(cache_path, jittor_path):
basenames = []
pyjt_names = []
for h in headers:
with open(h, 'r') as f:
with open(h, 'r', encoding='utf8') as f:
src = f.read()
bh = os.path.basename(h)
# jit_op_maker.h merge compile with var_holder.h
if bh == "var_holder.h": continue
if bh == "jit_op_maker.h":
with open(os.path.join(jittor_path, "src", "var_holder.h"), "r") as f:
with open(os.path.join(jittor_path, "src", "var_holder.h"), "r", encoding='utf8') as f:
src = f.read() + src
basename = bh.split(".")[0]
fname = "pyjt_"+basename+".cc"
@ -913,7 +913,7 @@ def compile(cache_path, jittor_path):
fname = os.path.join(cache_path, "gen", "pyjt_all.cc")
LOG.vvv(("write to", fname))
LOG.vvvv(code)
with open(fname, "w") as f:
with open(fname, "w", encoding='utf8') as f:
f.write(code)
pyjt_names.append(fname)
return pyjt_names

4
python/jittor/script/make_doc.py
View File

@ -1,7 +1,7 @@
import os
def fix_config(in_name, out_name, src_path, out_path):
data = open(in_name, 'r').readlines()
data = open(in_name, 'r', encoding='utf8').readlines()
out = []
for d in data:
if d.startswith('INPUT ='):
@ -9,7 +9,7 @@ def fix_config(in_name, out_name, src_path, out_path):
elif d.startswith('OUTPUT_DIRECTORY ='):
d = f'OUTPUT_DIRECTORY ={out_path}\n'
out.append(d)
f = open(out_name, 'w')
f = open(out_name, 'w', encoding='utf8')
f.writelines(out)
jt_path = os.getcwd()

8
python/jittor/src/jit_compiler.cc
View File

@ -153,7 +153,7 @@ jit_op_entry_t load_jit_lib(string name, string symbol_name="jit_entry") {
const char* msg = "";
LOGvv << "Opening jit lib:" << name;
#ifdef _WIN32
void* handle = (void*)LoadLibraryExA(name.c_str(), nullptr,
void* handle = (void*)LoadLibraryExA(Utf8ToGbk(name.c_str()).c_str(), nullptr,
LOAD_LIBRARY_SEARCH_DEFAULT_DIRS |
LOAD_LIBRARY_SEARCH_USER_DIRS);
#elif defined(__linux__)
@ -206,13 +206,15 @@ jit_op_entry_t compile(const string& jit_key, const string& src, const bool is_c
string jit_src_path = Op::get_filename_from_jit_key(jit_key, ".cc");
#ifdef _WIN32
string jit_lib_path = Op::get_filename_from_jit_key(jit_key, ".dll");
string jit_src_path2 = Utf8ToGbk(jit_src_path.c_str());
#else
string jit_lib_path = Op::get_filename_from_jit_key(jit_key, ".so");
string& jit_src_path2 = jit_src_path;
#endif
string other_src;
LOGvvv << "Generate" << jit_src_path >> "\n" >> src;
if (rewrite_op || !file_exist(jit_src_path))
write(jit_src_path, src);
if (rewrite_op || !file_exist(jit_src_path2))
write(jit_src_path2, src);
string cmd;
auto symbol_name = get_symbol_name(jit_key);

6
python/jittor/src/pyjt/py_converter.h
View File

@ -45,13 +45,7 @@ DEF_IS(string, PyObject*) to_py_object(const string& a) {
DEF_IS(string, string) from_py_object(PyObject* obj) {
Py_ssize_t size;
#ifdef _WIN32
PyObjHolder a(PyUnicode_AsEncodedString(obj, win_encode.c_str(), "strict"));
char* s;
auto ret = PyBytes_AsStringAndSize(a.obj, &s, &size);
#else
const char* s = PyUnicode_AsUTF8AndSize(obj, &size);
#endif
CHECK(s);
return string(s, size);
}

13
python/jittor/src/utils/cache_compile.cc
View File

@ -234,7 +234,15 @@ static inline bool is_full_path(const string& name) {
#endif
}
bool cache_compile(string cmd, const string& cache_path, const string& jittor_path) {
bool cache_compile(string cmd, const string& cache_path_, const string& jittor_path_) {
#ifdef _WIN32
cmd = Utf8ToGbk(cmd.c_str());
string cache_path = Utf8ToGbk(cache_path_.c_str());
string jittor_path = Utf8ToGbk(jittor_path_.c_str());
#else
const string& cache_path = cache_path_;
const string& jittor_path = jittor_path_;
#endif
vector<string> input_names;
map<string,vector<string>> extra;
string output_name;
@ -255,6 +263,9 @@ bool cache_compile(string cmd, const string& cache_path, const string& jittor_pa
continue;
processed.insert(input_names[i]);
auto src = read_all(input_names[i]);
#ifdef _WIN32
src = Utf8ToGbk(src.c_str());
#endif
auto back = input_names[i].back();
// *.lib
if (back == 'b') continue;

47
python/jittor/src/utils/log.cc
View File

@ -316,19 +316,13 @@ int register_sigaction() {
return 0;
}
#ifdef _WIN32
string win_encode;
#endif
static int log_init() {
#ifdef _WIN32
SetConsoleCP(CP_UTF8);
SetConsoleOutputCP(CP_UTF8);
#endif
register_sigaction();
std::atexit(log_exiting);
#ifdef _WIN32
if (getenv("JITTOR_ENCODE"))
win_encode = getenv("JITTOR_ENCODE");
else
win_encode = "gbk";
#endif
return 1;
}
@ -456,6 +450,39 @@ If you still have problems, please contact us:
}
#ifdef _WIN32
string GbkToUtf8(const char *src_str)
{
int len = MultiByteToWideChar(CP_ACP, 0, src_str, -1, NULL, 0);
wchar_t* wstr = new wchar_t[len + 1];
memset(wstr, 0, len + 1);
MultiByteToWideChar(CP_ACP, 0, src_str, -1, wstr, len);
len = WideCharToMultiByte(CP_UTF8, 0, wstr, -1, NULL, 0, NULL, NULL);
char* str = new char[len + 1];
memset(str, 0, len + 1);
WideCharToMultiByte(CP_UTF8, 0, wstr, -1, str, len, NULL, NULL);
string strTemp = str;
if (wstr) delete[] wstr;
if (str) delete[] str;
return strTemp;
}
string Utf8ToGbk(const char *src_str)
{
int len = MultiByteToWideChar(CP_UTF8, 0, src_str, -1, NULL, 0);
wchar_t* wszGBK = new wchar_t[len + 1];
memset(wszGBK, 0, len * 2 + 2);
MultiByteToWideChar(CP_UTF8, 0, src_str, -1, wszGBK, len);
len = WideCharToMultiByte(CP_ACP, 0, wszGBK, -1, NULL, 0, NULL, NULL);
char* szGBK = new char[len + 1];
memset(szGBK, 0, len + 1);
WideCharToMultiByte(CP_ACP, 0, wszGBK, -1, szGBK, len, NULL, NULL);
string strTemp(szGBK);
if (wszGBK) delete[] wszGBK;
if (szGBK) delete[] szGBK;
return strTemp;
}
int system_popen(const char *cmd, const char* cwd) {
HANDLE g_hChildStd_OUT_Rd = NULL;
HANDLE g_hChildStd_OUT_Wr = NULL;

8
python/jittor/src/utils/log.h
View File

@ -16,6 +16,10 @@ namespace jittor {
// define in tracer.cc
void print_trace();
void breakpoint();
#ifdef _WIN32
string GbkToUtf8(const char *src_str);
string Utf8ToGbk(const char *src_str);
#endif
constexpr int32_t basename_index(const char * const path, const int32_t index = 0, const int32_t slash_index = -1) {
return path[index]
@ -277,8 +281,4 @@ bool check_vlog(const char* fileline, int verbose);
void system_with_check(const char* cmd, const char* cwd=nullptr);
#ifdef _WIN32
extern string win_encode;
#endif
} // jittor

14
python/jittor/src/var_holder.h
View File

@ -181,12 +181,24 @@ struct VarHolder {
inline void set_requires_grad(bool flag) {
if (flag == get_requires_grad()) return;
if (flag)
_update(this);
start_grad();
else
stop_grad();
return;
}
/**
* enable the gradient calculation for the Var.
*/
// @pyjt(start_grad)
// @attrs(return_self)
inline VarHolder* start_grad() {
if (!var->dtype().is_float())
LOGw << "cannot enable grad of a non-float value:" << var;
_update(this);
return this;
}
// @pyjt(__get__uncertain_shape)
inline NanoVector uncertain_shape() {
return var->shape;

5
python/jittor/test/test_grad.py
View File

@ -143,6 +143,11 @@ class TestGrad(unittest.TestCase):
dx, = jt.grad(z, [x])
self.assertEqual(dx.data, 48)
def test_int_enable_grad(self):
a = jt.int([1,2,3])
a.requires_grad = True
a.start_grad()
def test_nth_grad(self):
x = jt.array(2.0)
y = x*x*x*x

6
python/jittor_utils/translator.py
View File

@ -23,7 +23,7 @@ def check_is_both(src):
for mdname in all_src_md:
print(mdname)
with open(mdname, "r") as f:
with open(mdname, "r", encoding='utf8') as f:
src = f.read()
src = src.split("```")
en_src = []
@ -47,8 +47,8 @@ for mdname in all_src_md:
cn_src.append("\n".join(cn_s))
en_src = "```".join(en_src)
cn_src = "```".join(cn_src)
with open(mdname.replace(".src.md", ".md"), 'w') as f:
with open(mdname.replace(".src.md", ".md"), 'w', encoding='utf8') as f:
f.write(en_src)
with open(mdname.replace(".src.md", ".cn.md"), 'w') as f:
with open(mdname.replace(".src.md", ".cn.md"), 'w', encoding='utf8') as f:
f.write(cn_src)